diff --git a/clang/docs/doxygen.cfg.in b/clang/docs/doxygen.cfg.in index 39a346409b93..95dcabca2cbd 100644 --- a/clang/docs/doxygen.cfg.in +++ b/clang/docs/doxygen.cfg.in @@ -1126,7 +1126,7 @@ HTML_TIMESTAMP = YES # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_DYNAMIC_SECTIONS = NO +HTML_DYNAMIC_SECTIONS = YES # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand @@ -1153,7 +1153,7 @@ HTML_INDEX_NUM_ENTRIES = 100 # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_DOCSET = NO +GENERATE_DOCSET = YES # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider @@ -1169,7 +1169,7 @@ DOCSET_FEEDNAME = "Doxygen generated docs" # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_BUNDLE_ID = clang # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style @@ -1177,7 +1177,7 @@ DOCSET_BUNDLE_ID = org.doxygen.Project # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_ID = Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. @@ -1252,7 +1252,7 @@ TOC_EXPAND = NO # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_QHP = @clang_doxygen_generate_qhp@ +GENERATE_QHP = YES # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to @@ -1267,7 +1267,7 @@ QCH_FILE = @clang_doxygen_qch_filename@ # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. -QHP_NAMESPACE = @clang_doxygen_qhp_namespace@ +QHP_NAMESPACE = clang # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual @@ -1337,7 +1337,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1468,7 +1468,7 @@ MATHJAX_CODEFILE = # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -SEARCHENGINE = @enable_searchengine@ +SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using Javascript. There @@ -2090,7 +2090,7 @@ CLASS_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -COLLABORATION_GRAPH = YES +COLLABORATION_GRAPH = NO # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h index 627e9025c112..20a1f9012293 100644 --- a/clang/include/clang/AST/OpenMPClause.h +++ b/clang/include/clang/AST/OpenMPClause.h @@ -8851,7 +8851,7 @@ struct TargetOMPContext final : public llvm::omp::OMPContext { const FunctionDecl *CurrentFunctionDecl, ArrayRef ConstructTraits); - virtual ~TargetOMPContext() = default; + virtual ~TargetOMPContext() override = default; /// See llvm::omp::OMPContext::matchesISATrait bool matchesISATrait(StringRef RawString) const override; diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h index 4175fe3072ab..87d0a2ca5626 100644 --- a/clang/include/clang/Basic/CodeGenOptions.h +++ b/clang/include/clang/Basic/CodeGenOptions.h @@ -398,6 +398,10 @@ class CodeGenOptions : public CodeGenOptionsBase { /// by sanitizer coverage pass. std::vector SanitizeCoverageAllowlistFiles; + /// Files whose declarations get rich debug info (types retained and + /// declaration-only subprograms emitted). + std::vector DebugInfoAllowedFiles; + /// The guard style used for stack protector to get a initial value, this /// value usually be gotten from TLS or get from __stack_chk_guard, or some /// other styles we may implement in the future. diff --git a/clang/include/clang/Basic/FileSystemOptions.h b/clang/include/clang/Basic/FileSystemOptions.h index 458af0c7b659..63fa3ba41b90 100644 --- a/clang/include/clang/Basic/FileSystemOptions.h +++ b/clang/include/clang/Basic/FileSystemOptions.h @@ -24,6 +24,11 @@ class FileSystemOptions { /// If set, paths are resolved as if the working directory was /// set to the value of WorkingDir. std::string WorkingDir; + + /// If set, file lookups that fail are retried with a case-insensitive + /// directory scan. Useful for compiling against SDKs whose header paths + /// use a different case than the #include directives. + bool CaseInsensitivePaths = false; }; } // end namespace clang diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 652c15afcce8..d73ffa11176e 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1282,6 +1282,11 @@ def fdebug_compilation_dir_EQ : Joined<["-"], "fdebug-compilation-dir=">, Group, Flags<[CC1Option, CC1AsOption, CoreOption]>, HelpText<"The compilation directory to embed in the debug info">, MarshallingInfoString>; +def fdebug_info_allowed_file_EQ : Joined<["-"], "fdebug-info-allowed-file=">, + Group, Flags<[CC1Option, CoreOption]>, + HelpText<"Emit rich debug info only for declarations from the given file " + "(repeatable)">, + MarshallingInfoStringVector>; def fdebug_compilation_dir : Separate<["-"], "fdebug-compilation-dir">, Group, Flags<[CC1Option, CC1AsOption, CoreOption]>, Alias; @@ -4376,6 +4381,10 @@ def working_directory : Separate<["-"], "working-directory">, Flags<[CC1Option]> MarshallingInfoString>; def working_directory_EQ : Joined<["-"], "working-directory=">, Flags<[CC1Option]>, Alias; +def fcase_insensitive_paths : Flag<["-"], "fcase-insensitive-paths">, + Flags<[CC1Option]>, + HelpText<"Retry failed file lookups with a case-insensitive directory scan">, + MarshallingInfoFlag>; // Double dash options, which are usually an alias for one of the previous // options. diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp index e8d0f20019eb..e923c0015d7f 100644 --- a/clang/lib/Basic/FileManager.cpp +++ b/clang/lib/Basic/FileManager.cpp @@ -577,6 +577,43 @@ FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize, isVolatile); } +/// Walk a path component-by-component, resolving each level with a +/// case-insensitive directory scan. Returns the real-cased path on success. +static std::optional +findPathCaseInsensitive(StringRef CaseInsensitivePath) { + using llvm::sys::fs::directory_iterator; + namespace path = llvm::sys::path; + + SmallString<256> Resolved = path::root_path(CaseInsensitivePath); + StringRef Relative = path::relative_path(CaseInsensitivePath); + if (Relative.empty()) + return std::nullopt; + + for (StringRef Component : + llvm::make_range(path::begin(Relative), path::end(Relative))) { + std::error_code EC; + std::string Match; + unsigned Hits = 0; + for (directory_iterator It(Resolved, EC), End; It != End && !EC; + It.increment(EC)) { + StringRef EntryName = path::filename(It->path()); + if (EntryName.equals_insensitive(Component)) { + if (Hits == 0) + Match = EntryName.str(); + ++Hits; + } + } + if (EC || Hits == 0) + return std::nullopt; + if (Hits > 1) + llvm::errs() << "warning: ambiguous case-insensitive match for '" + << Component << "' in '" << Resolved + << "'; picking '" << Match << "'\n"; + path::append(Resolved, Match); + } + return Resolved.str().str(); +} + /// getStatValue - Get the 'stat' information for the specified path, /// using the cache to accelerate it if possible. This returns true /// if the path points to a virtual file or does not exist, or returns @@ -587,15 +624,21 @@ FileManager::getStatValue(StringRef Path, llvm::vfs::Status &Status, bool isFile, std::unique_ptr *F) { // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be // absolute! - if (FileSystemOpts.WorkingDir.empty()) - return FileSystemStatCache::get(Path, Status, isFile, F, - StatCache.get(), *FS); - SmallString<128> FilePath(Path); - FixupRelativePath(FilePath); - - return FileSystemStatCache::get(FilePath.c_str(), Status, isFile, F, - StatCache.get(), *FS); + if (!FileSystemOpts.WorkingDir.empty()) + FixupRelativePath(FilePath); + + auto stat = [&](StringRef P) { + return FileSystemStatCache::get(P, Status, isFile, F, StatCache.get(), *FS); + }; + + auto EC = stat(FilePath); + if (!EC) + return EC; + if (FileSystemOpts.CaseInsensitivePaths) + if (auto Resolved = findPathCaseInsensitive(FilePath)) + return stat(*Resolved); + return EC; } std::error_code diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 3bde43cc1db3..65f8ab067742 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -4189,6 +4189,7 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, llvm::DINode::DIFlags Flags = llvm::DINode::FlagZero; llvm::DIFile *Unit = getOrCreateFile(Loc); bool IsDeclForCallSite = Fn ? true : false; + llvm::DIScope *FDContext = IsDeclForCallSite ? Unit : getDeclContextDescriptor(D); llvm::DINodeArray TParamsArray; @@ -4219,10 +4220,17 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D); llvm::DISubroutineType *STy = getOrCreateFunctionType(D, FnType, Unit); + // Only link to a prior declaration when creating a call-site entry. + // For standalone declarations the SP *is* the declaration; passing + // getFunctionDeclaration() here would let DISubprogram::get() unique + // the new node to the existing member SP, producing a self-referencing + // declaration link that causes infinite recursion in the DWARF emitter. + llvm::DISubprogram *Decl = + IsDeclForCallSite ? getFunctionDeclaration(D) : nullptr; llvm::DISubprogram *SP = DBuilder.createFunction(FDContext, Name, LinkageName, Unit, LineNo, STy, ScopeLine, Flags, SPFlags, TParamsArray.get(), - getFunctionDeclaration(D), nullptr, Annotations); + Decl, nullptr, Annotations); // Preserve btf_decl_tag attributes for parameters of extern functions // for BPF target. The parameters created in this loop are attached as @@ -4243,6 +4251,8 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, if (IsDeclForCallSite) Fn->setSubprogram(SP); + else if (CGM.getCodeGenOpts().hasMaybeUnusedDebugInfo()) + DBuilder.retainType(SP); DBuilder.finalizeSubprogram(SP); } diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 12d602fed693..9db3ec470738 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -6276,6 +6276,42 @@ void CodeGenModule::EmitDeclContext(const DeclContext *DC) { } } +/// Returns true if D comes from a file selected for rich debug info. +/// If -fdebug-info-allowed-file is not set, all declarations pass. +static bool isDeclInAllowedFiles(const Decl *D, CodeGenModule &CGM) { + const auto &Allowed = CGM.getCodeGenOpts().DebugInfoAllowedFiles; + if (Allowed.empty()) + return true; + auto &SM = CGM.getContext().getSourceManager(); + auto Filename = SM.getFilename(SM.getExpansionLoc(D->getLocation())); + if (Filename.empty()) + return false; + for (const auto &F : Allowed) + if (Filename == F) + return true; + return false; +} + +/// Emit a DISubprogram for a function declaration so its prototype ends +/// up in the debug output (LF_FUNC_ID in CodeView) even when the TU +/// has no definition. +static void emitFunctionDecl(CodeGenModule &CGM, const FunctionDecl *FD) { + // Only emit for declaration-only functions; definitions already get + // a DISubprogram through the normal codegen path. + if (FD->doesThisDeclarationHaveABody()) + return; + // Skip declarations whose type can't be represented in debug info + // (e.g. undeduced auto return types, dependent template types). + const auto *FPT = FD->getType()->getAs(); + if (!FPT || FPT->isDependentType() || FPT->isUndeducedType()) + return; + if (!isDeclInAllowedFiles(FD, CGM)) + return; + CGDebugInfo *DI = CGM.getModuleDebugInfo(); + if (DI) + DI->EmitFunctionDecl(GlobalDecl(FD), FD->getLocation(), QualType(FPT, 0)); +} + /// EmitTopLevelDecl - Emit code for a single top level declaration. void CodeGenModule::EmitTopLevelDecl(Decl *D) { // Ignore dependent declarations. @@ -6295,6 +6331,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { // Always provide some coverage mapping // even for the functions that aren't emitted. AddDeferredUnusedCoverageMapping(D); + emitFunctionDecl(*this, cast(D)); break; case Decl::CXXDeductionGuide: @@ -6331,7 +6368,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { case Decl::CXXRecord: { CXXRecordDecl *CRD = cast(D); if (CGDebugInfo *DI = getModuleDebugInfo()) { - if (CRD->hasDefinition()) + if (CRD->hasDefinition() && isDeclInAllowedFiles(D, *this)) DI->EmitAndRetainType(getContext().getRecordType(cast(D))); if (auto *ES = D->getASTContext().getExternalSource()) if (ES->hasExternalDefinitions(D) == ExternalASTSource::EK_Never) @@ -6553,19 +6590,22 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { case Decl::Typedef: case Decl::TypeAlias: // using foo = bar; [C++11] if (CGDebugInfo *DI = getModuleDebugInfo()) - DI->EmitAndRetainType( - getContext().getTypedefType(cast(D))); + if (isDeclInAllowedFiles(D, *this)) + DI->EmitAndRetainType( + getContext().getTypedefType(cast(D))); break; case Decl::Record: if (CGDebugInfo *DI = getModuleDebugInfo()) - if (cast(D)->getDefinition()) + if (cast(D)->getDefinition() && + isDeclInAllowedFiles(D, *this)) DI->EmitAndRetainType(getContext().getRecordType(cast(D))); break; case Decl::Enum: if (CGDebugInfo *DI = getModuleDebugInfo()) - if (cast(D)->getDefinition()) + if (cast(D)->getDefinition() && + isDeclInAllowedFiles(D, *this)) DI->EmitAndRetainType(getContext().getEnumType(cast(D))); break; diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index ec6860113b7e..ac223fe97e3f 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5743,6 +5743,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back(D.ResourceDir.c_str()); Args.AddLastArg(CmdArgs, options::OPT_working_directory); + Args.AddLastArg(CmdArgs, options::OPT_fcase_insensitive_paths); + Args.AddAllArgs(CmdArgs, options::OPT_fdebug_info_allowed_file_EQ); RenderARCMigrateToolOptions(D, Args, CmdArgs); diff --git a/clang/test/CodeGen/debug-info-allowed-file.c b/clang/test/CodeGen/debug-info-allowed-file.c new file mode 100644 index 000000000000..3ec308b10325 --- /dev/null +++ b/clang/test/CodeGen/debug-info-allowed-file.c @@ -0,0 +1,26 @@ +// Verify that -fdebug-info-allowed-file restricts rich debug-info emission to +// declarations from the listed files. + +// RUN: rm -rf %t && mkdir -p %t +// RUN: echo 'struct allowed_struct {}; void allowed_fn(int);' > %t/allowed.h +// RUN: echo 'struct excluded_struct {}; void excluded_fn(int);' > %t/excluded.h + +// With the filter, only decls from allowed.h appear in the debug output. +// RUN: %clang_cc1 -debug-info-kind=unused-types -emit-llvm -o - %s -I %t \ +// RUN: -fdebug-info-allowed-file=%t/allowed.h \ +// RUN: | FileCheck %s --check-prefix=FILTERED + +// Without the filter, both decls are emitted. +// RUN: %clang_cc1 -debug-info-kind=unused-types -emit-llvm -o - %s -I %t \ +// RUN: | FileCheck %s --check-prefix=UNFILTERED + +#include "allowed.h" +#include "excluded.h" + +// FILTERED-DAG: name: "allowed_struct" +// FILTERED-DAG: name: "allowed_fn" +// FILTERED-NOT: name: "excluded_struct" +// FILTERED-NOT: name: "excluded_fn" + +// UNFILTERED-DAG: name: "allowed_struct" +// UNFILTERED-DAG: name: "excluded_struct" diff --git a/clang/test/CodeGen/debug-info-codeview-emit-decl-subprogram.c b/clang/test/CodeGen/debug-info-codeview-emit-decl-subprogram.c new file mode 100644 index 000000000000..c8378f3419a1 --- /dev/null +++ b/clang/test/CodeGen/debug-info-codeview-emit-decl-subprogram.c @@ -0,0 +1,16 @@ +// Verify that a declaration-only function gets its DISubprogram retained +// and lowered to an LF_FUNC_ID record in the CodeView output. + +// REQUIRES: x86-registered-target + +// RUN: %clang_cc1 -triple x86_64-windows-msvc -gcodeview \ +// RUN: -debug-info-kind=unused-types -emit-obj -o %t.o %s +// RUN: llvm-readobj %t.o --codeview | FileCheck %s + +void declared_fn(int); + +// CHECK: FuncId ({{.*}}) { +// CHECK-NEXT: TypeLeafKind: LF_FUNC_ID +// CHECK: FunctionType: void (int) +// CHECK-NEXT: Name: declared_fn +// CHECK-NEXT: } diff --git a/clang/test/Driver/fcase-insensitive-paths.c b/clang/test/Driver/fcase-insensitive-paths.c new file mode 100644 index 000000000000..ef2f470a61ac --- /dev/null +++ b/clang/test/Driver/fcase-insensitive-paths.c @@ -0,0 +1,3 @@ +// RUN: %clang -### -fcase-insensitive-paths -c %s 2>&1 | FileCheck %s + +// CHECK: "-fcase-insensitive-paths" diff --git a/clang/test/Driver/fdebug-info-allowed-file.c b/clang/test/Driver/fdebug-info-allowed-file.c new file mode 100644 index 000000000000..e8678d18bef6 --- /dev/null +++ b/clang/test/Driver/fdebug-info-allowed-file.c @@ -0,0 +1,3 @@ +// RUN: %clang -### -fdebug-info-allowed-file=a.h -fdebug-info-allowed-file=b.h -c %s 2>&1 | FileCheck %s + +// CHECK: "-fdebug-info-allowed-file=a.h" "-fdebug-info-allowed-file=b.h" diff --git a/clang/test/Lexer/fcase-insensitive-paths.c b/clang/test/Lexer/fcase-insensitive-paths.c new file mode 100644 index 000000000000..6d92bcb46aef --- /dev/null +++ b/clang/test/Lexer/fcase-insensitive-paths.c @@ -0,0 +1,17 @@ +// Verify that -fcase-insensitive-paths retries failed header lookups with a +// case-insensitive directory scan. Only meaningful on a case-sensitive FS. + +// UNSUPPORTED: case-insensitive-filesystem + +// RUN: rm -rf %t && mkdir -p %t/inc +// RUN: echo 'int from_header;' > %t/inc/Header.h + +// Without the flag, the case-mismatched include fails. +// RUN: not %clang_cc1 -fsyntax-only -I %t/inc %s 2>&1 | FileCheck --check-prefix=OFF %s +// OFF: 'header.h' file not found + +// With the flag, the lookup succeeds. +// RUN: %clang_cc1 -fsyntax-only -fcase-insensitive-paths -I %t/inc %s + +#include "header.h" +int use_it(void) { return from_header; } diff --git a/clang/test/LibClang/symbols.test b/clang/test/LibClang/symbols.test index fd2ff8bc6cd4..5c48f4e62526 100644 --- a/clang/test/LibClang/symbols.test +++ b/clang/test/LibClang/symbols.test @@ -1,6 +1,6 @@ # Check that there are no unversioned clang symbols in libclang.so -RUN: llvm-nm -Dj --defined-only %libclang | grep -v -e '@@LLVM_[0-9]\+$' | not grep '^clang' +RUN: llvm-nm -Dj --defined-only %libclang | grep -v -e '@@LLVM_[0-9]\+$' | grep -vF '$plt' | grep -vF '$got' | not grep '^clang' # Check that here are no local clang_ symbols (ignoring symbols with .cold or # .localalias suffxies.) -RUN: llvm-nm %libclang | not grep '[a-z] clang_[^.]\+$' +RUN: llvm-nm %libclang | grep -vF '$plt' | grep -vF '$got' | not grep '[a-z] clang_[^.]\+$' diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c index c819a38553f3..174a4a624b34 100644 --- a/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/compiler-rt/lib/profile/InstrProfilingValue.c @@ -116,13 +116,6 @@ static ValueProfNode *allocateOneNode(void) { /* Early check to avoid value wrapping around. */ if (CurrentVNode + 1 > EndVNode) { - if (OutOfNodesWarnings++ < INSTR_PROF_MAX_VP_WARNS) { - PROF_WARN("Unable to track new values: %s. " - " Consider using option -mllvm -vp-counters-per-site= to " - "allocate more" - " value profile counters at compile time. \n", - "Running out of static counters"); - } return 0; } Node = COMPILER_RT_PTR_FETCH_ADD(ValueProfNode, CurrentVNode, 1); diff --git a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp index 2d206ed2dcbf..c42081869c7a 100644 --- a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp +++ b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp @@ -12,6 +12,7 @@ #include "TBAABuilder.h" #include "flang/Optimizer/Dialect/FIRType.h" +#include "llvm/ADT/TypeSwitch.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -159,9 +160,13 @@ void TBAABuilder::attachTBAATag(Operation *op, Type baseFIRType, else tbaaTagSym = getDataAccessTag(baseFIRType, accessFIRType, gep); - if (tbaaTagSym) - op->setAttr(LLVMDialect::getTBAAAttrName(), - ArrayAttr::get(op->getContext(), tbaaTagSym)); + if (!tbaaTagSym) + return; + + auto tbaaAttr = ArrayAttr::get(op->getContext(), tbaaTagSym); + llvm::TypeSwitch(op) + .Case([&](auto memOp) { memOp.setTbaaAttr(tbaaAttr); }) + .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); } } // namespace fir diff --git a/flang/test/Fir/tbaa.fir b/flang/test/Fir/tbaa.fir index ac94ea94cb3c..66261e6ee002 100644 --- a/flang/test/Fir/tbaa.fir +++ b/flang/test/Fir/tbaa.fir @@ -28,10 +28,10 @@ module { // CHECK: %[[VAL_4:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_5:.*]] = llvm.mlir.constant(10 : i32) : i32 // CHECK: %[[VAL_6:.*]] = llvm.getelementptr %[[VAL_0]][0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr>> -// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>> +// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>> // CHECK: %[[VAL_8:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_9:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_10:.*]] = llvm.load %[[VAL_9]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_10:.*]] = llvm.load %[[VAL_9]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_11:.*]] = llvm.mul %[[VAL_4]], %[[VAL_10]] : i64 // CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_11]], %[[VAL_8]] : i64 // CHECK: %[[VAL_13:.*]] = llvm.bitcast %[[VAL_7]] : !llvm.ptr> to !llvm.ptr @@ -40,11 +40,11 @@ module { // CHECK: %[[VAL_16:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_17:.*]] = llvm.mlir.constant(-1 : i32) : i32 // CHECK: %[[VAL_18:.*]] = llvm.getelementptr %[[VAL_0]][0, 8] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_20:.*]] = llvm.getelementptr %[[VAL_0]][0, 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_21:.*]] = llvm.load %[[VAL_20]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_21:.*]] = llvm.load %[[VAL_20]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_0]][0, 4] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_24:.*]] = llvm.mlir.undef : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> // CHECK: %[[VAL_25:.*]] = llvm.insertvalue %[[VAL_21]], %[[VAL_24]][1] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> // CHECK: %[[VAL_26:.*]] = llvm.mlir.constant(20180515 : i32) : i32 @@ -64,15 +64,15 @@ module { // CHECK: %[[VAL_40:.*]] = llvm.insertvalue %[[VAL_39]], %[[VAL_38]][7] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> // CHECK: %[[VAL_41:.*]] = llvm.bitcast %[[VAL_15]] : !llvm.ptr> to !llvm.ptr> // CHECK: %[[VAL_42:.*]] = llvm.insertvalue %[[VAL_41]], %[[VAL_40]][0] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)> -// CHECK: llvm.store %[[VAL_42]], %[[VAL_2]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>> +// CHECK: llvm.store %[[VAL_42]], %[[VAL_2]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>> // CHECK: %[[VAL_43:.*]] = llvm.getelementptr %[[VAL_2]][0, 4] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_44:.*]] = llvm.load %[[VAL_43]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_44:.*]] = llvm.load %[[VAL_43]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_45:.*]] = llvm.icmp "eq" %[[VAL_44]], %[[VAL_3]] : i8 // CHECK: llvm.cond_br %[[VAL_45]], ^bb1, ^bb2 // CHECK: ^bb1: // CHECK: %[[VAL_46:.*]] = llvm.getelementptr %[[VAL_2]][0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_47:.*]] = llvm.load %[[VAL_46]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> -// CHECK: llvm.store %[[VAL_5]], %[[VAL_47]] {llvm.tbaa = [@__flang_tbaa::@[[DATAT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_47:.*]] = llvm.load %[[VAL_46]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: llvm.store %[[VAL_5]], %[[VAL_47]] {tbaa = [@__flang_tbaa::@[[DATAT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: llvm.br ^bb2 // CHECK: ^bb2: // CHECK: llvm.return @@ -133,24 +133,24 @@ module { // CHECK: %[[VAL_8:.*]] = llvm.mlir.addressof @_QQcl.2E2F64756D6D792E66393000 : !llvm.ptr> // CHECK: %[[VAL_9:.*]] = llvm.bitcast %[[VAL_8]] : !llvm.ptr> to !llvm.ptr // CHECK: %[[VAL_10:.*]] = llvm.call @_FortranAioBeginExternalListOutput(%[[VAL_6]], %[[VAL_9]], %[[VAL_5]]) {fastmathFlags = #llvm.fastmath} : (i32, !llvm.ptr, i32) -> !llvm.ptr -// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_7]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> -// CHECK: llvm.store %[[VAL_11]], %[[VAL_3]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> +// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_7]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> +// CHECK: llvm.store %[[VAL_11]], %[[VAL_3]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> // CHECK: %[[VAL_12:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>, i64) -> !llvm.ptr -// CHECK: %[[VAL_13:.*]] = llvm.load %[[VAL_12]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_13:.*]] = llvm.load %[[VAL_12]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_14:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>, i64) -> !llvm.ptr -// CHECK: %[[VAL_15:.*]] = llvm.load %[[VAL_14]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_15:.*]] = llvm.load %[[VAL_14]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_16:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, %[[VAL_4]], 2] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>, i64) -> !llvm.ptr -// CHECK: %[[VAL_17:.*]] = llvm.load %[[VAL_16]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_17:.*]] = llvm.load %[[VAL_16]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_18:.*]] = llvm.getelementptr %[[VAL_3]][0, 8] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_19:.*]] = llvm.load %[[VAL_18]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_20:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_21:.*]] = llvm.mlir.constant(-1 : i32) : i32 // CHECK: %[[VAL_22:.*]] = llvm.getelementptr %[[VAL_3]][0, 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_23:.*]] = llvm.load %[[VAL_22]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_24:.*]] = llvm.getelementptr %[[VAL_3]][0, 4] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_25:.*]] = llvm.load %[[VAL_24]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_25:.*]] = llvm.load %[[VAL_24]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_26:.*]] = llvm.getelementptr %[[VAL_3]][0, 8] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_27:.*]] = llvm.load %[[VAL_26]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_27:.*]] = llvm.load %[[VAL_26]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_28:.*]] = llvm.mlir.undef : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_29:.*]] = llvm.insertvalue %[[VAL_23]], %[[VAL_28]][1] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_30:.*]] = llvm.mlir.constant(20180515 : i32) : i32 @@ -169,13 +169,13 @@ module { // CHECK: %[[VAL_43:.*]] = llvm.bitcast %[[VAL_27]] : !llvm.ptr to !llvm.ptr // CHECK: %[[VAL_44:.*]] = llvm.insertvalue %[[VAL_43]], %[[VAL_42]][8] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_45:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, 0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_46:.*]] = llvm.load %[[VAL_45]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_46:.*]] = llvm.load %[[VAL_45]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_47:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, 0, 1] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_48:.*]] = llvm.load %[[VAL_47]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_48:.*]] = llvm.load %[[VAL_47]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_49:.*]] = llvm.getelementptr %[[VAL_3]][0, 7, 0, 2] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr -// CHECK: %[[VAL_50:.*]] = llvm.load %[[VAL_49]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr +// CHECK: %[[VAL_50:.*]] = llvm.load %[[VAL_49]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr // CHECK: %[[VAL_51:.*]] = llvm.getelementptr %[[VAL_3]][0, 0] : (!llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>>) -> !llvm.ptr>> -// CHECK: %[[VAL_52:.*]] = llvm.load %[[VAL_51]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>> +// CHECK: %[[VAL_52:.*]] = llvm.load %[[VAL_51]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>> // CHECK: %[[VAL_53:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_54:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_55:.*]] = llvm.icmp "eq" %[[VAL_48]], %[[VAL_53]] : i64 @@ -185,7 +185,7 @@ module { // CHECK: %[[VAL_59:.*]] = llvm.insertvalue %[[VAL_50]], %[[VAL_58]][7, 0, 2] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_60:.*]] = llvm.bitcast %[[VAL_52]] : !llvm.ptr> to !llvm.ptr> // CHECK: %[[VAL_61:.*]] = llvm.insertvalue %[[VAL_60]], %[[VAL_59]][0] : !llvm.struct<(ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> -// CHECK: llvm.store %[[VAL_61]], %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> +// CHECK: llvm.store %[[VAL_61]], %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> // CHECK: %[[VAL_62:.*]] = llvm.bitcast %[[VAL_1]] : !llvm.ptr>, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)>> to !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>> // CHECK: %[[VAL_63:.*]] = llvm.call @_FortranAioOutputDescriptor(%[[VAL_10]], %[[VAL_62]]) {fastmathFlags = #llvm.fastmath} : (!llvm.ptr, !llvm.ptr>, i64, i32, i8, i8, i8, i8, ptr, array<1 x i64>)>>) -> i1 // CHECK: %[[VAL_64:.*]] = llvm.call @_FortranAioEndIoStatement(%[[VAL_10]]) {fastmathFlags = #llvm.fastmath} : (!llvm.ptr) -> i32 @@ -253,7 +253,7 @@ func.func @tbaa(%arg0: !fir.box>) -> i32 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i32 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 3] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: llvm.return %[[VAL_2]] : i32 // CHECK: } @@ -275,7 +275,7 @@ func.func @tbaa(%arg0: !fir.box>) -> i1 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i1 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 3] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: %[[VAL_3:.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: %[[VAL_4:.*]] = llvm.icmp "ne" %[[VAL_2]], %[[VAL_3]] : i32 // CHECK: llvm.return %[[VAL_4]] : i1 @@ -299,7 +299,7 @@ func.func @tbaa(%arg0: !fir.box) -> i32 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i32 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 1] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: llvm.return %[[VAL_2]] : i32 // CHECK: } @@ -321,7 +321,7 @@ func.func @tbaa(%arg0: !fir.box>) -> i1 { // CHECK-LABEL: llvm.func @tbaa( // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> i1 { // CHECK: %[[VAL_1:.*]] = llvm.getelementptr %[[VAL_0]][0, 5] : (!llvm.ptr, i64, i32, i8, i8, i8, i8)>>) -> !llvm.ptr -// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_2:.*]] = llvm.load %[[VAL_1]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: %[[VAL_3:.*]] = llvm.mlir.constant(2 : i32) : i32 // CHECK: %[[VAL_4:.*]] = llvm.and %[[VAL_2]], %[[VAL_3]] : i32 // CHECK: %[[VAL_5:.*]] = llvm.mlir.constant(0 : i32) : i32 @@ -353,11 +353,11 @@ func.func @tbaa(%arg0: !fir.box>) { // CHECK: %[[VAL_4:.*]] = llvm.sub %[[VAL_1]], %[[VAL_2]] : i64 // CHECK: %[[VAL_5:.*]] = llvm.mul %[[VAL_4]], %[[VAL_2]] : i64 // CHECK: %[[VAL_6:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>>) -> !llvm.ptr -// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr +// CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {tbaa = [@__flang_tbaa::@[[BOXT:tag_[0-9]*]]]} : !llvm.ptr // CHECK: %[[VAL_8:.*]] = llvm.mul %[[VAL_5]], %[[VAL_7]] : i64 // CHECK: %[[VAL_9:.*]] = llvm.add %[[VAL_8]], %[[VAL_3]] : i64 // CHECK: %[[VAL_10:.*]] = llvm.getelementptr %[[VAL_0]][0, 0] : (!llvm.ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>>) -> !llvm.ptr> -// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_10]] {llvm.tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> +// CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_10]] {tbaa = [@__flang_tbaa::@[[BOXT]]]} : !llvm.ptr> // CHECK: %[[VAL_12:.*]] = llvm.bitcast %[[VAL_11]] : !llvm.ptr to !llvm.ptr // CHECK: %[[VAL_13:.*]] = llvm.getelementptr %[[VAL_12]]{{\[}}%[[VAL_9]]] : (!llvm.ptr, i64) -> !llvm.ptr // CHECK: %[[VAL_14:.*]] = llvm.bitcast %[[VAL_13]] : !llvm.ptr to !llvm.ptr diff --git a/libcxx/include/cstdlib b/libcxx/include/cstdlib index 25c9de5165c2..e69ea70e6042 100644 --- a/libcxx/include/cstdlib +++ b/libcxx/include/cstdlib @@ -85,6 +85,7 @@ void *aligned_alloc(size_t alignment, size_t size); // C11 #include <__config> #include +#include #ifndef _LIBCPP_STDLIB_H # error tried including but didn't find libc++'s header. \ @@ -98,6 +99,10 @@ void *aligned_alloc(size_t alignment, size_t size); // C11 # pragma GCC system_header #endif +extern "C" inline void *aligned_alloc(size_t alignment, size_t size) { + return memalign(alignment, size); +} + _LIBCPP_BEGIN_NAMESPACE_STD using ::size_t _LIBCPP_USING_IF_EXISTS; diff --git a/libcxx/test/configs/llvm-libc++-shared.cfg.in b/libcxx/test/configs/llvm-libc++-shared.cfg.in index 7228c7e8d467..709eb17cbd76 100644 --- a/libcxx/test/configs/llvm-libc++-shared.cfg.in +++ b/libcxx/test/configs/llvm-libc++-shared.cfg.in @@ -10,7 +10,7 @@ config.substitutions.append(('%{compile_flags}', '-nostdinc++ -I %{include} -I %{target-include} -I %{libcxx}/test/support' )) config.substitutions.append(('%{link_flags}', - '-nostdlib++ -L %{lib} -Wl,-rpath,%{lib} -lc++ -pthread' + '-nostdlib++ -L %{lib} -Wl,-rpath,%{lib} -lc++ -lc++abi -pthread' )) config.substitutions.append(('%{exec}', '%{executor} --execdir %T -- ' diff --git a/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp b/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp index a02a5bbbdbcc..8639c43fbe50 100644 --- a/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp +++ b/libcxx/test/std/depr/depr.c.headers/stdlib_h.aligned_alloc.compile.pass.cpp @@ -21,7 +21,7 @@ // ::aligned_alloc is available starting with Android P (API 28) // XFAIL: target={{.+}}-android{{(eabi)?(21|22|23|24|25|26|27)}} -#include +#include #include static_assert(std::is_same::value, ""); diff --git a/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp b/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp index 016e51d99c59..f093107f3767 100644 --- a/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp +++ b/libcxx/test/std/depr/depr.c.headers/uchar_h.compile.pass.cpp @@ -14,6 +14,8 @@ // The system-provided seems to be broken on AIX // XFAIL: LIBCXX-AIX-FIXME +// XFAIL: target={{.+}} + // #include diff --git a/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp b/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp index 37c7b6723478..03250ae8218a 100644 --- a/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp +++ b/libcxx/test/std/language.support/support.runtime/ctime.timespec.compile.pass.cpp @@ -20,6 +20,8 @@ // ::timespec_get is available starting with Android Q (API 29) // XFAIL: target={{.+}}-android{{(eabi)?(21|22|23|24|25|26|27|28)}} +// XFAIL: target={{.+}} + #include #include diff --git a/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp b/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp index 909f2c025bdf..69160aedebe4 100644 --- a/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp +++ b/libcxx/test/std/strings/c.strings/cuchar.compile.pass.cpp @@ -14,6 +14,8 @@ // The system-provided seems to be broken on AIX // XFAIL: LIBCXX-AIX-FIXME +// XFAIL: target={{.+}} + // #include diff --git a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake index d760eee07958..d92c58f1966f 100644 --- a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake +++ b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake @@ -291,6 +291,9 @@ function(llvm_ExternalProject_Add name source_dir) list(APPEND compiler_args -DCMAKE_ASM_COMPILER_TARGET=${ARG_TARGET_TRIPLE}) endif() + list(APPEND compiler_args -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}) + list(APPEND compiler_args -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) + ExternalProject_Add(${name} DEPENDS ${ARG_DEPENDS} llvm-config ${name}-clobber diff --git a/llvm/cmake/modules/VersionFromVCS.cmake b/llvm/cmake/modules/VersionFromVCS.cmake index 18edbeabe3e4..23e80a91503d 100644 --- a/llvm/cmake/modules/VersionFromVCS.cmake +++ b/llvm/cmake/modules/VersionFromVCS.cmake @@ -5,7 +5,7 @@ function(get_source_info path revision repository) find_package(Git QUIET) - if(GIT_FOUND) + if(FALSE) execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --git-dir WORKING_DIRECTORY ${path} RESULT_VARIABLE git_result diff --git a/llvm/docs/ProgrammersManual.rst b/llvm/docs/ProgrammersManual.rst index d1b3744e22da..99391b43de87 100644 --- a/llvm/docs/ProgrammersManual.rst +++ b/llvm/docs/ProgrammersManual.rst @@ -2040,6 +2040,16 @@ that are not simple pointers (use :ref:`SmallPtrSet ` for pointers). Note that DenseSet has the same requirements for the value type that :ref:`DenseMap ` has. +.. _dss_radixtree: + +llvm/ADT/RadixTree.h +^^^^^^^^^^^^^^^^^^^^ + +``RadixTree`` is a trie-based data structure that stores range-like keys and +their associated values. It is particularly efficient for storing keys that +share common prefixes, as it can compress these prefixes to save memory. It +supports efficient search of matching prefixes. + .. _dss_sparseset: llvm/ADT/SparseSet.h diff --git a/llvm/docs/doxygen.cfg.in b/llvm/docs/doxygen.cfg.in index b19ca6d21580..a856380fb30b 100644 --- a/llvm/docs/doxygen.cfg.in +++ b/llvm/docs/doxygen.cfg.in @@ -1127,7 +1127,7 @@ HTML_TIMESTAMP = YES # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_DYNAMIC_SECTIONS = NO +HTML_DYNAMIC_SECTIONS = YES # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand @@ -1154,7 +1154,7 @@ HTML_INDEX_NUM_ENTRIES = 100 # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_DOCSET = NO +GENERATE_DOCSET = YES # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider @@ -1170,7 +1170,7 @@ DOCSET_FEEDNAME = "Doxygen generated docs" # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_BUNDLE_ID = llvm # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style @@ -1178,7 +1178,7 @@ DOCSET_BUNDLE_ID = org.doxygen.Project # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_ID = Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. @@ -1268,7 +1268,7 @@ QCH_FILE = @llvm_doxygen_qch_filename@ # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. -QHP_NAMESPACE = @llvm_doxygen_qhp_namespace@ +QHP_NAMESPACE = llvm # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual @@ -1338,7 +1338,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1469,7 +1469,7 @@ MATHJAX_CODEFILE = # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -SEARCHENGINE = @enable_searchengine@ +SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using Javascript. There @@ -2091,7 +2091,7 @@ CLASS_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -COLLABORATION_GRAPH = YES +COLLABORATION_GRAPH = NO # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. diff --git a/llvm/include/llvm/ADT/ADL.h b/llvm/include/llvm/ADT/ADL.h new file mode 100644 index 000000000000..812d9a4b52d8 --- /dev/null +++ b/llvm/include/llvm/ADT/ADL.h @@ -0,0 +1,135 @@ +//===- llvm/ADT/ADL.h - Argument dependent lookup utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_ADL_H +#define LLVM_ADT_ADL_H + +#include +#include +#include + +namespace llvm { + +// Only used by compiler if both template types are the same. Useful when +// using SFINAE to test for the existence of member functions. +template struct SameType; + +namespace adl_detail { + +using std::begin; + +template +constexpr auto begin_impl(RangeT &&range) + -> decltype(begin(std::forward(range))) { + return begin(std::forward(range)); +} + +using std::end; + +template +constexpr auto end_impl(RangeT &&range) + -> decltype(end(std::forward(range))) { + return end(std::forward(range)); +} + +using std::rbegin; + +template +constexpr auto rbegin_impl(RangeT &&range) + -> decltype(rbegin(std::forward(range))) { + return rbegin(std::forward(range)); +} + +using std::rend; + +template +constexpr auto rend_impl(RangeT &&range) + -> decltype(rend(std::forward(range))) { + return rend(std::forward(range)); +} + +using std::swap; + +template +constexpr void swap_impl(T &&lhs, + T &&rhs) noexcept(noexcept(swap(std::declval(), + std::declval()))) { + swap(std::forward(lhs), std::forward(rhs)); +} + +using std::size; + +template +constexpr auto size_impl(RangeT &&range) + -> decltype(size(std::forward(range))) { + return size(std::forward(range)); +} + +} // end namespace adl_detail + +/// Returns the begin iterator to \p range using `std::begin` and +/// function found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_begin(RangeT &&range) + -> decltype(adl_detail::begin_impl(std::forward(range))) { + return adl_detail::begin_impl(std::forward(range)); +} + +/// Returns the end iterator to \p range using `std::end` and +/// functions found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_end(RangeT &&range) + -> decltype(adl_detail::end_impl(std::forward(range))) { + return adl_detail::end_impl(std::forward(range)); +} + +/// Returns the reverse-begin iterator to \p range using `std::rbegin` and +/// function found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_rbegin(RangeT &&range) + -> decltype(adl_detail::rbegin_impl(std::forward(range))) { + return adl_detail::rbegin_impl(std::forward(range)); +} + +/// Returns the reverse-end iterator to \p range using `std::rend` and +/// functions found through Argument-Dependent Lookup (ADL). +template +constexpr auto adl_rend(RangeT &&range) + -> decltype(adl_detail::rend_impl(std::forward(range))) { + return adl_detail::rend_impl(std::forward(range)); +} + +/// Swaps \p lhs with \p rhs using `std::swap` and functions found through +/// Argument-Dependent Lookup (ADL). +template +constexpr void adl_swap(T &&lhs, T &&rhs) noexcept( + noexcept(adl_detail::swap_impl(std::declval(), std::declval()))) { + adl_detail::swap_impl(std::forward(lhs), std::forward(rhs)); +} + +/// Returns the size of \p range using `std::size` and functions found through +/// Argument-Dependent Lookup (ADL). +template +constexpr auto adl_size(RangeT &&range) + -> decltype(adl_detail::size_impl(std::forward(range))) { + return adl_detail::size_impl(std::forward(range)); +} + +namespace detail { + +template +using IterOfRange = decltype(adl_begin(std::declval())); + +template +using ValueOfRange = + std::remove_reference_t()))>; + +} // namespace detail +} // namespace llvm + +#endif // LLVM_ADT_ADL_H diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h index c0e2d13c2939..b86a03c42433 100644 --- a/llvm/include/llvm/ADT/APFloat.h +++ b/llvm/include/llvm/ADT/APFloat.h @@ -640,21 +640,14 @@ class DoubleAPFloat final : public APFloatBase { DoubleAPFloat(DoubleAPFloat &&RHS); DoubleAPFloat &operator=(const DoubleAPFloat &RHS); - - DoubleAPFloat &operator=(DoubleAPFloat &&RHS) { - if (this != &RHS) { - this->~DoubleAPFloat(); - new (this) DoubleAPFloat(std::move(RHS)); - } - return *this; - } + inline DoubleAPFloat &operator=(DoubleAPFloat &&RHS); bool needsCleanup() const { return Floats != nullptr; } - APFloat &getFirst() { return Floats[0]; } - const APFloat &getFirst() const { return Floats[0]; } - APFloat &getSecond() { return Floats[1]; } - const APFloat &getSecond() const { return Floats[1]; } + inline APFloat &getFirst(); + inline const APFloat &getFirst() const; + inline APFloat &getSecond(); + inline const APFloat &getSecond() const; opStatus add(const DoubleAPFloat &RHS, roundingMode RM); opStatus subtract(const DoubleAPFloat &RHS, roundingMode RM); @@ -1358,6 +1351,27 @@ inline APFloat maximum(const APFloat &A, const APFloat &B) { return A < B ? B : A; } +// We want the following functions to be available in the header for inlining. +// We cannot define them inline in the class definition of `DoubleAPFloat` +// because doing so would instantiate `std::unique_ptr` before +// `APFloat` is defined, and that would be undefined behavior. +namespace detail { + +DoubleAPFloat &DoubleAPFloat::operator=(DoubleAPFloat &&RHS) { + if (this != &RHS) { + this->~DoubleAPFloat(); + new (this) DoubleAPFloat(std::move(RHS)); + } + return *this; +} + +APFloat &DoubleAPFloat::getFirst() { return Floats[0]; } +const APFloat &DoubleAPFloat::getFirst() const { return Floats[0]; } +APFloat &DoubleAPFloat::getSecond() { return Floats[1]; } +const APFloat &DoubleAPFloat::getSecond() const { return Floats[1]; } + +} // namespace detail + } // namespace llvm #undef APFLOAT_DISPATCH_ON_SEMANTICS diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h index 07ac1768ea27..582ad6fcbfc8 100644 --- a/llvm/include/llvm/ADT/GenericCycleImpl.h +++ b/llvm/include/llvm/ADT/GenericCycleImpl.h @@ -31,8 +31,8 @@ namespace llvm { -template -bool GenericCycle::contains(const GenericCycle *C) const { +template +bool GenericCycle::contains(const GenericCycle *C) const { if (!C) return false; @@ -43,14 +43,15 @@ bool GenericCycle::contains(const GenericCycle *C) const { return this == C; } -template -void GenericCycle::getExitBlocks( +template +void GenericCycle::getExitBlocks( SmallVectorImpl &TmpStorage) const { TmpStorage.clear(); size_t NumExitBlocks = 0; for (BlockT *Block : blocks()) { - llvm::append_range(TmpStorage, successors(Block)); + auto BlockSuccessors = llvm::children(Block); + llvm::append_range(TmpStorage, BlockSuccessors); for (size_t Idx = NumExitBlocks, End = TmpStorage.size(); Idx < End; ++Idx) { @@ -66,8 +67,8 @@ void GenericCycle::getExitBlocks( } } -template -auto GenericCycle::getCyclePreheader() const -> BlockT * { +template +auto GenericCycle::getCyclePreheader() const -> BlockT * { BlockT *Predecessor = getCyclePredecessor(); if (!Predecessor) return nullptr; @@ -84,8 +85,9 @@ auto GenericCycle::getCyclePreheader() const -> BlockT * { return Predecessor; } -template -auto GenericCycle::getCyclePredecessor() const -> BlockT * { +template +auto GenericCycle::getCyclePredecessor() const + -> BlockT * { if (!isReducible()) return nullptr; @@ -93,7 +95,8 @@ auto GenericCycle::getCyclePredecessor() const -> BlockT * { // Loop over the predecessors of the header node... BlockT *Header = getHeader(); - for (const auto Pred : predecessors(Header)) { + auto BlockPredecessors = llvm::children>(Header); + for (const auto Pred : BlockPredecessors) { if (!contains(Pred)) { if (Out && Out != Pred) return nullptr; @@ -105,9 +108,9 @@ auto GenericCycle::getCyclePredecessor() const -> BlockT * { } /// \brief Helper class for computing cycle information. -template class GenericCycleInfoCompute { +template class GenericCycleInfoCompute { using BlockT = typename ContextT::BlockT; - using CycleInfoT = GenericCycleInfo; + using CycleInfoT = GenericCycleInfo; using CycleT = typename CycleInfoT::CycleT; CycleInfoT &Info; @@ -143,9 +146,9 @@ template class GenericCycleInfoCompute { void dfs(BlockT *EntryBlock); }; -template -auto GenericCycleInfo::getTopLevelParentCycle(BlockT *Block) - -> CycleT * { +template +auto GenericCycleInfo::getTopLevelParentCycle( + BlockT *Block) -> CycleT * { auto Cycle = BlockMapTopLevel.find(Block); if (Cycle != BlockMapTopLevel.end()) return Cycle->second; @@ -161,9 +164,9 @@ auto GenericCycleInfo::getTopLevelParentCycle(BlockT *Block) return C; } -template -void GenericCycleInfo::moveTopLevelCycleToNewParent(CycleT *NewParent, - CycleT *Child) { +template +void GenericCycleInfo::moveTopLevelCycleToNewParent( + CycleT *NewParent, CycleT *Child) { assert((!Child->ParentCycle && !NewParent->ParentCycle) && "NewParent and Child must be both top level cycle!\n"); auto &CurrentContainer = @@ -186,8 +189,8 @@ void GenericCycleInfo::moveTopLevelCycleToNewParent(CycleT *NewParent, } /// \brief Main function of the cycle info computations. -template -void GenericCycleInfoCompute::run(BlockT *EntryBlock) { +template +void GenericCycleInfoCompute::run(BlockT *EntryBlock) { LLVM_DEBUG(errs() << "Entry block: " << Info.Context.print(EntryBlock) << "\n"); dfs(EntryBlock); @@ -197,7 +200,9 @@ void GenericCycleInfoCompute::run(BlockT *EntryBlock) { for (BlockT *HeaderCandidate : llvm::reverse(BlockPreorder)) { const DFSInfo CandidateInfo = BlockDFSInfo.lookup(HeaderCandidate); - for (BlockT *Pred : predecessors(HeaderCandidate)) { + auto BlockPredecessors = + llvm::children>(HeaderCandidate); + for (BlockT *Pred : BlockPredecessors) { const DFSInfo PredDFSInfo = BlockDFSInfo.lookup(Pred); if (CandidateInfo.isAncestorOf(PredDFSInfo)) Worklist.push_back(Pred); @@ -221,7 +226,8 @@ void GenericCycleInfoCompute::run(BlockT *EntryBlock) { LLVM_DEBUG(errs() << " block " << Info.Context.print(Block) << ": "); bool IsEntry = false; - for (BlockT *Pred : predecessors(Block)) { + auto BlockPredecessors = llvm::children>(Block); + for (BlockT *Pred : BlockPredecessors) { const DFSInfo PredDFSInfo = BlockDFSInfo.lookup(Pred); if (CandidateInfo.isAncestorOf(PredDFSInfo)) { Worklist.push_back(Pred); @@ -286,8 +292,9 @@ void GenericCycleInfoCompute::run(BlockT *EntryBlock) { } /// \brief Recompute depth values of \p SubTree and all descendants. -template -void GenericCycleInfoCompute::updateDepth(CycleT *SubTree) { +template +void GenericCycleInfoCompute::updateDepth( + CycleT *SubTree) { for (CycleT *Cycle : depth_first(SubTree)) Cycle->Depth = Cycle->ParentCycle ? Cycle->ParentCycle->Depth + 1 : 1; } @@ -295,8 +302,8 @@ void GenericCycleInfoCompute::updateDepth(CycleT *SubTree) { /// \brief Compute a DFS of basic blocks starting at the function entry. /// /// Fills BlockDFSInfo with start/end counters and BlockPreorder. -template -void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { +template +void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { SmallVector DFSTreeStack; SmallVector TraverseStack; unsigned Counter = 0; @@ -315,7 +322,8 @@ void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { << TraverseStack.size() << "\n"); DFSTreeStack.emplace_back(TraverseStack.size()); - llvm::append_range(TraverseStack, successors(Block)); + auto BlockSuccessors = llvm::children(Block); + llvm::append_range(TraverseStack, BlockSuccessors); bool Added = BlockDFSInfo.try_emplace(Block, ++Counter).second; (void)Added; @@ -345,16 +353,17 @@ void GenericCycleInfoCompute::dfs(BlockT *EntryBlock) { } /// \brief Reset the object to its initial state. -template void GenericCycleInfo::clear() { +template +void GenericCycleInfo::clear() { TopLevelCycles.clear(); BlockMap.clear(); BlockMapTopLevel.clear(); } /// \brief Compute the cycle info for a function. -template -void GenericCycleInfo::compute(FunctionT &F) { - GenericCycleInfoCompute Compute(*this); +template +void GenericCycleInfo::compute(FunctionT &F) { + GenericCycleInfoCompute Compute(*this); Context.setFunction(F); LLVM_DEBUG(errs() << "Computing cycles for function: " << F.getName() @@ -368,8 +377,8 @@ void GenericCycleInfo::compute(FunctionT &F) { /// /// \returns the innermost cycle containing \p Block or nullptr if /// it is not contained in any cycle. -template -auto GenericCycleInfo::getCycle(const BlockT *Block) const +template +auto GenericCycleInfo::getCycle(const BlockT *Block) const -> CycleT * { auto MapIt = BlockMap.find(Block); if (MapIt != BlockMap.end()) @@ -381,8 +390,9 @@ auto GenericCycleInfo::getCycle(const BlockT *Block) const /// /// \returns the depth for the innermost cycle containing \p Block or 0 if it is /// not contained in any cycle. -template -unsigned GenericCycleInfo::getCycleDepth(const BlockT *Block) const { +template +unsigned GenericCycleInfo::getCycleDepth( + const BlockT *Block) const { CycleT *Cycle = getCycle(Block); if (!Cycle) return 0; @@ -394,8 +404,8 @@ unsigned GenericCycleInfo::getCycleDepth(const BlockT *Block) const { /// /// Note that this does \em not check that cycles are really cycles in the CFG, /// or that the right set of cycles in the CFG were found. -template -bool GenericCycleInfo::validateTree() const { +template +bool GenericCycleInfo::validateTree() const { DenseSet Blocks; DenseSet Entries; @@ -458,8 +468,8 @@ bool GenericCycleInfo::validateTree() const { #endif /// \brief Print the cycle info. -template -void GenericCycleInfo::print(raw_ostream &Out) const { +template +void GenericCycleInfo::print(raw_ostream &Out) const { for (const auto *TLC : toplevel_cycles()) { for (const CycleT *Cycle : depth_first(TLC)) { for (unsigned I = 0; I < Cycle->Depth; ++I) diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h index 63db9eb9a601..9cc182f32a7a 100644 --- a/llvm/include/llvm/ADT/GenericCycleInfo.h +++ b/llvm/include/llvm/ADT/GenericCycleInfo.h @@ -34,6 +34,7 @@ #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Printable.h" #include "llvm/Support/raw_ostream.h" @@ -41,16 +42,19 @@ namespace llvm { -template class GenericCycleInfo; -template class GenericCycleInfoCompute; +template +class GenericCycleInfo; +template +class GenericCycleInfoCompute; /// A possibly irreducible generalization of a \ref Loop. -template class GenericCycle { +template +class GenericCycle { public: using BlockT = typename ContextT::BlockT; using FunctionT = typename ContextT::FunctionT; - template friend class GenericCycleInfo; - template friend class GenericCycleInfoCompute; + template friend class GenericCycleInfo; + template friend class GenericCycleInfoCompute; private: /// The parent cycle. Is null for the root "cycle". Top-level cycles point @@ -223,13 +227,15 @@ template class GenericCycle { }; /// \brief Cycle information for a function. -template class GenericCycleInfo { +/// \tparam GraphView A graph view which is used, through `GraphTraits`, as the +/// the graph on which `GenericCycleInfo` is computed. +template class GenericCycleInfo { public: using BlockT = typename ContextT::BlockT; - using CycleT = GenericCycle; + using CycleT = GenericCycle; using FunctionT = typename ContextT::FunctionT; - template friend class GenericCycle; - template friend class GenericCycleInfoCompute; + template friend class GenericCycle; + template friend class GenericCycleInfoCompute; private: ContextT Context; @@ -343,14 +349,16 @@ template struct CycleGraphTraits { // Return total number of nodes in the graph }; -template -struct GraphTraits *> - : CycleGraphTraits *, - typename GenericCycle::const_child_iterator> {}; -template -struct GraphTraits *> - : CycleGraphTraits *, - typename GenericCycle::const_child_iterator> {}; +template +struct GraphTraits *> + : CycleGraphTraits< + const GenericCycle *, + typename GenericCycle::const_child_iterator> {}; +template +struct GraphTraits *> + : CycleGraphTraits< + GenericCycle *, + typename GenericCycle::const_child_iterator> {}; } // namespace llvm diff --git a/llvm/include/llvm/ADT/IntervalMap.h b/llvm/include/llvm/ADT/IntervalMap.h index c68d816a8e7d..c2dbc4fffc90 100644 --- a/llvm/include/llvm/ADT/IntervalMap.h +++ b/llvm/include/llvm/ADT/IntervalMap.h @@ -1215,7 +1215,9 @@ IntervalMapImpl::IdxPair IntervalMap:: branchRoot(unsigned Position) { using namespace IntervalMapImpl; // How many external leaf nodes to hold RootLeaf+1? - const unsigned Nodes = RootLeaf::Capacity / Leaf::Capacity + 1; + using CapacityType = decltype(RootLeaf::Capacity); + const unsigned Nodes = (RootLeaf::Capacity + / static_cast(Leaf::Capacity) + 1); // Compute element distribution among new nodes. unsigned size[Nodes]; @@ -1256,7 +1258,9 @@ IntervalMapImpl::IdxPair IntervalMap:: splitRoot(unsigned Position) { using namespace IntervalMapImpl; // How many external leaf nodes to hold RootBranch+1? - const unsigned Nodes = RootBranch::Capacity / Branch::Capacity + 1; + using CapacityType = decltype(RootBranch::Capacity); + const unsigned Nodes = (RootBranch::Capacity + / static_cast(Branch::Capacity) + 1); // Compute element distribution among new nodes. unsigned Size[Nodes]; diff --git a/llvm/include/llvm/ADT/RadixTree.h b/llvm/include/llvm/ADT/RadixTree.h new file mode 100644 index 000000000000..87e2a3ebecc0 --- /dev/null +++ b/llvm/include/llvm/ADT/RadixTree.h @@ -0,0 +1,351 @@ +//===-- llvm/ADT/RadixTree.h - Radix Tree implementation --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +//===----------------------------------------------------------------------===// +// +// This file implements a Radix Tree. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_RADIXTREE_H +#define LLVM_ADT_RADIXTREE_H + +#include "llvm/ADT/ADL.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" +#include +#include +#include +#include +#include +#include +#include + +namespace llvm { + +/// \brief A Radix Tree implementation. +/// +/// A Radix Tree (also known as a compact prefix tree or radix trie) is a +/// data structure that stores a dynamic set or associative array where keys +/// are strings and values are associated with these keys. Unlike a regular +/// trie, the edges of a radix tree can be labeled with sequences of characters +/// as well as single characters. This makes radix trees more efficient for +/// storing sparse data sets, where many nodes in a regular trie would have +/// only one child. +/// +/// This implementation supports arbitrary key types that can be iterated over +/// (e.g., `std::string`, `std::vector`, `ArrayRef`). The key type +/// must provide `begin()` and `end()` for iteration. +/// +/// The tree stores `std::pair` as its value type. +/// +/// Example usage: +/// \code +/// llvm::RadixTree Tree; +/// Tree.emplace("apple", 1); +/// Tree.emplace("grapefruit", 2); +/// Tree.emplace("grape", 3); +/// +/// // Find prefixes +/// for (const auto &[Key, Value] : Tree.find_prefixes("grapefruit juice")) { +/// // pair will be {"grape", 3} +/// // pair will be {"grapefruit", 2} +/// llvm::outs() << Key << ": " << Value << "\n"; +/// } +/// +/// // Iterate over all elements +/// for (const auto &[Key, Value] : Tree) +/// llvm::outs() << Key << ": " << Value << "\n"; +/// \endcode +/// +/// \note +/// The `RadixTree` takes ownership of the `KeyType` and `T` objects +/// inserted into it. When an element is removed or the tree is destroyed, +/// these objects will be destructed. +/// However, if `KeyType` is a reference-like type, e.g., StringRef or range, +/// the user must guarantee that the referenced data has a lifetime longer than +/// the tree. +template class RadixTree { +public: + using key_type = KeyType; + using mapped_type = T; + using value_type = std::pair; + +private: + using KeyConstIteratorType = + decltype(adl_begin(std::declval())); + using KeyConstIteratorRangeType = iterator_range; + using KeyValueType = + remove_cvref_t()))>; + using ContainerType = std::list; + + /// Represents an internal node in the Radix Tree. + struct Node { + KeyConstIteratorRangeType Key{KeyConstIteratorType{}, + KeyConstIteratorType{}}; + std::vector Children; + + /// An iterator to the value associated with this node. + /// + /// If this node does not have a value (i.e., it's an internal node that + /// only serves as a path to other values), this iterator will be equal + /// to default constructed `ContainerType::iterator()`. + std::optional Value; + + /// The first character of the Key. Used for fast child lookup. + KeyValueType KeyFront; + + Node() = default; + Node(const KeyConstIteratorRangeType &Key) + : Key(Key), KeyFront(*Key.begin()) { + assert(!Key.empty()); + } + + Node(Node &&) = default; + Node &operator=(Node &&) = default; + + Node(const Node &) = delete; + Node &operator=(const Node &) = delete; + + const Node *findChild(const KeyConstIteratorRangeType &Key) const { + if (Key.empty()) + return nullptr; + for (const Node &Child : Children) { + assert(!Child.Key.empty()); // Only root can be empty. + if (Child.KeyFront == *Key.begin()) + return &Child; + } + return nullptr; + } + + Node *findChild(const KeyConstIteratorRangeType &Query) { + const Node *This = this; + return const_cast(This->findChild(Query)); + } + + size_t countNodes() const { + size_t R = 1; + for (const Node &C : Children) + R += C.countNodes(); + return R; + } + + /// + /// Splits the current node into two. + /// + /// This function is used when a new key needs to be inserted that shares + /// a common prefix with the current node's key, but then diverges. + /// The current `Key` is truncated to the common prefix, and a new child + /// node is created for the remainder of the original node's `Key`. + /// + /// \param SplitPoint An iterator pointing to the character in the current + /// `Key` where the split should occur. + void split(KeyConstIteratorType SplitPoint) { + Node Child(make_range(SplitPoint, Key.end())); + Key = make_range(Key.begin(), SplitPoint); + + Children.swap(Child.Children); + std::swap(Value, Child.Value); + + Children.emplace_back(std::move(Child)); + } + }; + + /// Root always corresponds to the empty key, which is the shortest possible + /// prefix for everything. + Node Root; + ContainerType KeyValuePairs; + + /// Finds or creates a new tail or leaf node corresponding to the `Key`. + Node &findOrCreate(KeyConstIteratorRangeType Key) { + Node *Curr = &Root; + if (Key.empty()) + return *Curr; + + for (;;) { + auto [I1, I2] = llvm::mismatch(Key, Curr->Key); + Key = make_range(I1, Key.end()); + + if (I2 != Curr->Key.end()) { + // Match is partial. Either query is too short, or there is mismatching + // character. Split either way, and put new node in between of the + // current and its children. + Curr->split(I2); + + // Split was caused by mismatch, so `findChild` would fail. + break; + } + + Node *Child = Curr->findChild(Key); + if (!Child) + break; + + // Move to child with the same first character. + Curr = Child; + } + + if (Key.empty()) { + // The current node completely matches the key, return it. + return *Curr; + } + + // `Key` is a suffix of original `Key` unmatched by path from the `Root` to + // the `Curr`, and we have no candidate in the children to match more. + // Create a new one. + return Curr->Children.emplace_back(Key); + } + + /// + /// An iterator for traversing prefixes search results. + /// + /// This iterator is used by `find_prefixes` to traverse the tree and find + /// elements that are prefixes to the given key. It's a forward iterator. + /// + /// \tparam MappedType The type of the value pointed to by the iterator. + /// This will be `value_type` for non-const iterators + /// and `const value_type` for const iterators. + template + class IteratorImpl + : public iterator_facade_base, + std::forward_iterator_tag, MappedType> { + const Node *Curr = nullptr; + KeyConstIteratorRangeType Query{KeyConstIteratorType{}, + KeyConstIteratorType{}}; + + void findNextValid() { + while (Curr && !Curr->Value.has_value()) + advance(); + } + + void advance() { + assert(Curr); + if (Query.empty()) { + Curr = nullptr; + return; + } + + Curr = Curr->findChild(Query); + if (!Curr) { + Curr = nullptr; + return; + } + + auto [I1, I2] = llvm::mismatch(Query, Curr->Key); + if (I2 != Curr->Key.end()) { + Curr = nullptr; + return; + } + Query = make_range(I1, Query.end()); + } + + friend class RadixTree; + IteratorImpl(const Node *C, const KeyConstIteratorRangeType &Q) + : Curr(C), Query(Q) { + findNextValid(); + } + + public: + IteratorImpl() = default; + + MappedType &operator*() const { return **Curr->Value; } + + IteratorImpl &operator++() { + advance(); + findNextValid(); + return *this; + } + + bool operator==(const IteratorImpl &Other) const { + return Curr == Other.Curr; + } + }; + +public: + RadixTree() = default; + RadixTree(RadixTree &&) = default; + RadixTree &operator=(RadixTree &&) = default; + + using prefix_iterator = IteratorImpl; + using const_prefix_iterator = IteratorImpl; + + using iterator = typename ContainerType::iterator; + using const_iterator = typename ContainerType::const_iterator; + + /// Returns true if the tree is empty. + bool empty() const { return KeyValuePairs.empty(); } + + /// Returns the number of elements in the tree. + size_t size() const { return KeyValuePairs.size(); } + + /// Returns the number of nodes in the tree. + /// + /// This function counts all internal nodes in the tree. It can be useful for + /// understanding the memory footprint or complexity of the tree structure. + size_t countNodes() const { return Root.countNodes(); } + + /// Returns an iterator to the first element. + iterator begin() { return KeyValuePairs.begin(); } + const_iterator begin() const { return KeyValuePairs.begin(); } + + /// Returns an iterator to the end of the tree. + iterator end() { return KeyValuePairs.end(); } + const_iterator end() const { return KeyValuePairs.end(); } + + /// Constructs and inserts a new element into the tree. + /// + /// This function constructs an element in place within the tree. If an + /// element with the same key already exists, the insertion fails and the + /// function returns an iterator to the existing element along with `false`. + /// Otherwise, the new element is inserted and the function returns an + /// iterator to the new element along with `true`. + /// + /// \param Key The key of the element to construct. + /// \param Args Arguments to forward to the constructor of the mapped_type. + /// \return A pair consisting of an iterator to the inserted element (or to + /// the element that prevented insertion) and a boolean value + /// indicating whether the insertion took place. + template + std::pair emplace(key_type &&Key, Ts &&...Args) { + // We want to make new `Node` to refer key in the container, not the one + // from the argument. + // FIXME: Determine that we need a new node, before expanding + // `KeyValuePairs`. + const value_type &NewValue = KeyValuePairs.emplace_front( + std::move(Key), T(std::forward(Args)...)); + Node &Node = findOrCreate(NewValue.first); + bool HasValue = Node.Value.has_value(); + if (!HasValue) + Node.Value = KeyValuePairs.begin(); + else + KeyValuePairs.pop_front(); + return {*Node.Value, !HasValue}; + } + + /// + /// Finds all elements whose keys are prefixes of the given `Key`. + /// + /// This function returns an iterator range over all elements in the tree + /// whose keys are prefixes of the provided `Key`. For example, if the tree + /// contains "abcde", "abc", "abcdefgh", and `Key` is "abcde", this function + /// would return iterators to "abcde" and "abc". + /// + /// \param Key The key to search for prefixes of. + /// \return An `iterator_range` of `const_prefix_iterator`s, allowing + /// iteration over the found prefix elements. + /// \note The returned iterators reference the `Key` provided by the caller. + /// The caller must ensure that `Key` remains valid for the lifetime + /// of the iterators. + iterator_range + find_prefixes(const key_type &Key) const { + return iterator_range{ + const_prefix_iterator(&Root, KeyConstIteratorRangeType(Key)), + const_prefix_iterator{}}; + } +}; + +} // namespace llvm + +#endif // LLVM_ADT_RADIXTREE_H diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index 79b145632d5a..8c0421daf2da 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -17,6 +17,7 @@ #ifndef LLVM_ADT_STLEXTRAS_H #define LLVM_ADT_STLEXTRAS_H +#include "llvm/ADT/ADL.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/STLFunctionalExtras.h" @@ -46,21 +47,6 @@ namespace llvm { -// Only used by compiler if both template types are the same. Useful when -// using SFINAE to test for the existence of member functions. -template struct SameType; - -namespace detail { - -template -using IterOfRange = decltype(std::begin(std::declval())); - -template -using ValueOfRange = - std::remove_reference_t()))>; - -} // end namespace detail - //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// @@ -333,48 +319,6 @@ template class Callable { } // namespace callable_detail -namespace adl_detail { - -using std::begin; - -template -decltype(auto) adl_begin(ContainerTy &&container) { - return begin(std::forward(container)); -} - -using std::end; - -template -decltype(auto) adl_end(ContainerTy &&container) { - return end(std::forward(container)); -} - -using std::swap; - -template -void adl_swap(T &&lhs, T &&rhs) noexcept(noexcept(swap(std::declval(), - std::declval()))) { - swap(std::forward(lhs), std::forward(rhs)); -} - -} // end namespace adl_detail - -template -decltype(auto) adl_begin(ContainerTy &&container) { - return adl_detail::adl_begin(std::forward(container)); -} - -template -decltype(auto) adl_end(ContainerTy &&container) { - return adl_detail::adl_end(std::forward(container)); -} - -template -void adl_swap(T &&lhs, T &&rhs) noexcept( - noexcept(adl_detail::adl_swap(std::declval(), std::declval()))) { - adl_detail::adl_swap(std::forward(lhs), std::forward(rhs)); -} - /// Returns true if the given container only contains a single element. template bool hasSingleElement(ContainerTy &&C) { auto B = std::begin(C), E = std::end(C); @@ -461,32 +405,23 @@ class mapped_iterator_base } }; -/// Helper to determine if type T has a member called rbegin(). -template class has_rbegin_impl { - using yes = char[1]; - using no = char[2]; - - template - static yes& test(Inner *I, decltype(I->rbegin()) * = nullptr); - - template - static no& test(...); - -public: - static const bool value = sizeof(test(nullptr)) == sizeof(yes); -}; +namespace detail { +template +using check_has_free_function_rbegin = + decltype(adl_rbegin(std::declval())); -/// Metafunction to determine if T& or T has a member called rbegin(). -template -struct has_rbegin : has_rbegin_impl> {}; +template +static constexpr bool HasFreeFunctionRBegin = + is_detected::value; +} // namespace detail // Returns an iterator_range over the given container which iterates in reverse. template auto reverse(ContainerTy &&C) { - if constexpr (has_rbegin::value) - return make_range(C.rbegin(), C.rend()); + if constexpr (detail::HasFreeFunctionRBegin) + return make_range(adl_rbegin(C), adl_rend(C)); else - return make_range(std::make_reverse_iterator(std::end(C)), - std::make_reverse_iterator(std::begin(C))); + return make_range(std::make_reverse_iterator(adl_end(C)), + std::make_reverse_iterator(adl_begin(C))); } /// An iterator adaptor that filters the elements of given inner iterators. @@ -1315,15 +1250,11 @@ class indexed_accessor_range_base { } /// Compare this range with another. - template - friend bool operator==(const indexed_accessor_range_base &lhs, - const OtherT &rhs) { - return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); + template bool operator==(const OtherT &rhs) const { + return std::equal(begin(), end(), rhs.begin(), rhs.end()); } - template - friend bool operator!=(const indexed_accessor_range_base &lhs, - const OtherT &rhs) { - return !(lhs == rhs); + template bool operator!=(const OtherT &rhs) const { + return !(*this == rhs); } /// Return the size of this range. @@ -1468,6 +1399,10 @@ template auto make_second_range(ContainerTy &&c) { }); } +template auto make_address_range(R &&Range) { + return map_range(Range, [](auto &Element) { return ∈ }); +} + //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// @@ -1944,6 +1879,19 @@ auto upper_bound(R &&Range, T &&Value, Compare C) { std::forward(Value), C); } +/// Wrapper function around std::mismatch to find the first mismatching pair +/// in two ranges. +template +auto mismatch(R1 &&Range1, R2 &&Range2) { + return std::mismatch(adl_begin(Range1), adl_end(Range1), adl_begin(Range2), + adl_end(Range2)); +} + +/// Returns the size of a range, i.e., the number of elements. +template auto range_size(R &&Range) { + return std::distance(adl_begin(Range), adl_end(Range)); +} + template void stable_sort(R &&Range) { std::stable_sort(adl_begin(Range), adl_end(Range)); diff --git a/llvm/include/llvm/ADT/SetOperations.h b/llvm/include/llvm/ADT/SetOperations.h index c9462f077dc8..52aced706893 100644 --- a/llvm/include/llvm/ADT/SetOperations.h +++ b/llvm/include/llvm/ADT/SetOperations.h @@ -45,6 +45,25 @@ void set_intersect(S1Ty &S1, const S2Ty &S2) { } } +template +S1Ty set_intersection_impl(const S1Ty &S1, const S2Ty &S2) { + S1Ty Result; + for (typename S1Ty::const_iterator SI = S1.begin(), SE = S1.end(); SI != SE; + ++SI) + if (S2.count(*SI)) + Result.insert(*SI); + return Result; +} + +/// set_intersection(A, B) - Return A ^ B +template +S1Ty set_intersection(const S1Ty &S1, const S2Ty &S2) { + if (S1.size() < S2.size()) + return set_intersection_impl(S1, S2); + else + return set_intersection_impl(S2, S1); +} + /// set_difference(A, B) - Return A - B /// template @@ -66,6 +85,19 @@ void set_subtract(S1Ty &S1, const S2Ty &S2) { S1.erase(*SI); } +/// set_subtract(A, B, C, D) - Compute A := A - B, set C to the elements of B +/// removed from A (A ^ B), and D to the elements of B not found in and removed +/// from A (B - A). +template +void set_subtract(S1Ty &S1, const S2Ty &S2, S1Ty &Removed, S1Ty &Remaining) { + for (typename S2Ty::const_iterator SI = S2.begin(), SE = S2.end(); SI != SE; + ++SI) + if (S1.erase(*SI)) + Removed.insert(*SI); + else + Remaining.insert(*SI); +} + /// set_is_subset(A, B) - Return true iff A in B /// template diff --git a/llvm/include/llvm/ADT/SetVector.h b/llvm/include/llvm/ADT/SetVector.h index 37509e28f891..5aa8771044d1 100644 --- a/llvm/include/llvm/ADT/SetVector.h +++ b/llvm/include/llvm/ADT/SetVector.h @@ -68,6 +68,12 @@ class SetVector { return std::move(vector_); } + /// Clear the SetVector and return the underlying set. + Set takeSet() { + vector_.clear(); + return std::move(set_); + } + /// Determine if the SetVector is empty or not. bool empty() const { return vector_.empty(); diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 2d2f0bedfe1f..8d34c31aae39 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -54,6 +55,9 @@ namespace llvm { using iterator = const char *; using const_iterator = const char *; using size_type = size_t; + using value_type = char; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; private: /// The start of the string, in an external buffer. @@ -112,6 +116,14 @@ namespace llvm { iterator end() const { return Data + Length; } + reverse_iterator rbegin() const { + return std::make_reverse_iterator(end()); + } + + reverse_iterator rend() const { + return std::make_reverse_iterator(begin()); + } + const unsigned char *bytes_begin() const { return reinterpret_cast(begin()); } diff --git a/llvm/include/llvm/ADT/iterator_range.h b/llvm/include/llvm/ADT/iterator_range.h index a9b46a3aa45b..4f33a27adadd 100644 --- a/llvm/include/llvm/ADT/iterator_range.h +++ b/llvm/include/llvm/ADT/iterator_range.h @@ -18,10 +18,22 @@ #ifndef LLVM_ADT_ITERATOR_RANGE_H #define LLVM_ADT_ITERATOR_RANGE_H +#include "llvm/ADT/ADL.h" +#include #include namespace llvm { +template +struct explicitly_convertable : std::false_type {}; + +template +struct explicitly_convertable< + From, To, + std::void_t( + std::declval>()))>> : std::true_type { +}; + /// A range adaptor for a pair of iterators. /// /// This just wraps two iterators into a range-compatible interface. Nothing @@ -31,12 +43,12 @@ class iterator_range { IteratorT begin_iterator, end_iterator; public: - //TODO: Add SFINAE to test that the Container's iterators match the range's - // iterators. - template + template , IteratorT>::value> * = nullptr> iterator_range(Container &&c) - //TODO: Consider ADL/non-member begin/end calls. - : begin_iterator(c.begin()), end_iterator(c.end()) {} + : begin_iterator(adl_begin(std::forward(c))), + end_iterator(adl_end(std::forward(c))) {} iterator_range(IteratorT begin_iterator, IteratorT end_iterator) : begin_iterator(std::move(begin_iterator)), end_iterator(std::move(end_iterator)) {} @@ -46,6 +58,9 @@ class iterator_range { bool empty() const { return begin_iterator == end_iterator; } }; +template +iterator_range(Container &&) -> iterator_range>; + /// Convenience function for iterating over sub-ranges. /// /// This provides a bit of syntactic sugar to make using sub-ranges diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index 953e15e358f1..1a8a87d9e360 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -588,7 +588,73 @@ class AAResults { MemoryEffects getMemoryEffects(const CallBase *Call, AAQueryInfo &AAQI); private: - class Concept; + /// A private abstract base class describing the concept of an individual alias + /// analysis implementation. + /// + /// This interface is implemented by any \c Model instantiation. It is also the + /// interface which a type used to instantiate the model must provide. + /// + /// All of these methods model methods by the same name in the \c + /// AAResults class. Only differences and specifics to how the + /// implementations are called are documented here. + class Concept { + public: + virtual ~Concept() = 0; + + //===--------------------------------------------------------------------===// + /// \name Alias Queries + /// @{ + + /// The main low level interface to the alias analysis implementation. + /// Returns an AliasResult indicating whether the two pointers are aliased to + /// each other. This is the interface that must be implemented by specific + /// alias analysis implementations. + virtual AliasResult alias(const MemoryLocation &LocA, + const MemoryLocation &LocB, AAQueryInfo &AAQI, + const Instruction *CtxI) = 0; + + /// @} + //===--------------------------------------------------------------------===// + /// \name Simple mod/ref information + /// @{ + + /// Returns a bitmask that should be unconditionally applied to the ModRef + /// info of a memory location. This allows us to eliminate Mod and/or Ref from + /// the ModRef info based on the knowledge that the memory location points to + /// constant and/or locally-invariant memory. + virtual ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, + AAQueryInfo &AAQI, + bool IgnoreLocals) = 0; + + /// Get the ModRef info associated with a pointer argument of a callsite. The + /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note + /// that these bits do not necessarily account for the overall behavior of + /// the function, but rather only provide additional per-argument + /// information. + virtual ModRefInfo getArgModRefInfo(const CallBase *Call, + unsigned ArgIdx) = 0; + + /// Return the behavior of the given call site. + virtual MemoryEffects getMemoryEffects(const CallBase *Call, + AAQueryInfo &AAQI) = 0; + + /// Return the behavior when calling the given function. + virtual MemoryEffects getMemoryEffects(const Function *F) = 0; + + /// getModRefInfo (for call sites) - Return information about whether + /// a particular call site modifies or reads the specified memory location. + virtual ModRefInfo getModRefInfo(const CallBase *Call, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) = 0; + + /// Return information about whether two call sites may refer to the same set + /// of memory locations. See the AA documentation for details: + /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo + virtual ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI) = 0; + + /// @} + }; template class Model; @@ -665,74 +731,6 @@ class BatchAAResults { /// pointer or reference. using AliasAnalysis = AAResults; -/// A private abstract base class describing the concept of an individual alias -/// analysis implementation. -/// -/// This interface is implemented by any \c Model instantiation. It is also the -/// interface which a type used to instantiate the model must provide. -/// -/// All of these methods model methods by the same name in the \c -/// AAResults class. Only differences and specifics to how the -/// implementations are called are documented here. -class AAResults::Concept { -public: - virtual ~Concept() = 0; - - //===--------------------------------------------------------------------===// - /// \name Alias Queries - /// @{ - - /// The main low level interface to the alias analysis implementation. - /// Returns an AliasResult indicating whether the two pointers are aliased to - /// each other. This is the interface that must be implemented by specific - /// alias analysis implementations. - virtual AliasResult alias(const MemoryLocation &LocA, - const MemoryLocation &LocB, AAQueryInfo &AAQI, - const Instruction *CtxI) = 0; - - /// @} - //===--------------------------------------------------------------------===// - /// \name Simple mod/ref information - /// @{ - - /// Returns a bitmask that should be unconditionally applied to the ModRef - /// info of a memory location. This allows us to eliminate Mod and/or Ref from - /// the ModRef info based on the knowledge that the memory location points to - /// constant and/or locally-invariant memory. - virtual ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, - AAQueryInfo &AAQI, - bool IgnoreLocals) = 0; - - /// Get the ModRef info associated with a pointer argument of a callsite. The - /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note - /// that these bits do not necessarily account for the overall behavior of - /// the function, but rather only provide additional per-argument - /// information. - virtual ModRefInfo getArgModRefInfo(const CallBase *Call, - unsigned ArgIdx) = 0; - - /// Return the behavior of the given call site. - virtual MemoryEffects getMemoryEffects(const CallBase *Call, - AAQueryInfo &AAQI) = 0; - - /// Return the behavior when calling the given function. - virtual MemoryEffects getMemoryEffects(const Function *F) = 0; - - /// getModRefInfo (for call sites) - Return information about whether - /// a particular call site modifies or reads the specified memory location. - virtual ModRefInfo getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc, - AAQueryInfo &AAQI) = 0; - - /// Return information about whether two call sites may refer to the same set - /// of memory locations. See the AA documentation for details: - /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo - virtual ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, - AAQueryInfo &AAQI) = 0; - - /// @} -}; - /// A private class template which derives from \c Concept and wraps some other /// type. /// diff --git a/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h b/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h index 542a741ee07e..bcb891fb8d0b 100644 --- a/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h +++ b/llvm/include/llvm/Analysis/IteratedDominanceFrontier.h @@ -73,7 +73,7 @@ ChildrenGetterTy::get(const NodeRef &N) { return {Children.begin(), Children.end()}; } - return GD->template getChildren(N); + return GD->template getChildren(N); } } // end of namespace IDFCalculatorDetail diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h index b77a335d1ee6..e1752a835366 100644 --- a/llvm/include/llvm/Analysis/LoopInfo.h +++ b/llvm/include/llvm/Analysis/LoopInfo.h @@ -63,7 +63,16 @@ class MDNode; class MemorySSAUpdater; class ScalarEvolution; class raw_ostream; -template class DominatorTreeBase; + +template +using DTIdentityView = X; + +template class View> +class DominatorTreeOnView; + +template +using DominatorTreeBase = DominatorTreeOnView; + template class LoopInfoBase; template class LoopBase; diff --git a/llvm/include/llvm/BinaryFormat/MachO.h b/llvm/include/llvm/BinaryFormat/MachO.h index d51af31fb14f..f7941e0441db 100644 --- a/llvm/include/llvm/BinaryFormat/MachO.h +++ b/llvm/include/llvm/BinaryFormat/MachO.h @@ -232,6 +232,7 @@ enum DataRegionType { }; enum RebaseType { + REBASE_TYPE_INVALID = 0u, REBASE_TYPE_POINTER = 1u, REBASE_TYPE_TEXT_ABSOLUTE32 = 2u, REBASE_TYPE_TEXT_PCREL32 = 3u @@ -252,6 +253,7 @@ enum RebaseOpcode { }; enum BindType { + BIND_TYPE_INVALID = 0u, BIND_TYPE_POINTER = 1u, BIND_TYPE_TEXT_ABSOLUTE32 = 2u, BIND_TYPE_TEXT_PCREL32 = 3u diff --git a/llvm/include/llvm/CodeGen/LiveIntervalCalc.h b/llvm/include/llvm/CodeGen/LiveIntervalCalc.h index b8d67ef8f4e1..62485547c2e9 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervalCalc.h +++ b/llvm/include/llvm/CodeGen/LiveIntervalCalc.h @@ -21,7 +21,14 @@ namespace llvm { -template class DomTreeNodeBase; +template +using DTIdentityView = X; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; using MachineDomTreeNode = DomTreeNodeBase; diff --git a/llvm/include/llvm/CodeGen/LiveRangeCalc.h b/llvm/include/llvm/CodeGen/LiveRangeCalc.h index 895ecff18f89..b941b2dd4312 100644 --- a/llvm/include/llvm/CodeGen/LiveRangeCalc.h +++ b/llvm/include/llvm/CodeGen/LiveRangeCalc.h @@ -35,13 +35,21 @@ namespace llvm { -template class DomTreeNodeBase; +template +using DTIdentityView = X; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; + +using MachineDomTreeNode = DomTreeNodeBase; + class MachineDominatorTree; class MachineFunction; class MachineRegisterInfo; -using MachineDomTreeNode = DomTreeNodeBase; - class LiveRangeCalc { const MachineFunction *MF = nullptr; const MachineRegisterInfo *MRI = nullptr; diff --git a/llvm/include/llvm/CodeGen/MachineDominators.h b/llvm/include/llvm/CodeGen/MachineDominators.h index 30c18ef410fa..958139a981a0 100644 --- a/llvm/include/llvm/CodeGen/MachineDominators.h +++ b/llvm/include/llvm/CodeGen/MachineDominators.h @@ -37,9 +37,9 @@ inline void DominatorTreeBase::addRoot( this->Roots.push_back(MBB); } -extern template class DomTreeNodeBase; -extern template class DominatorTreeBase; // DomTree -extern template class DominatorTreeBase; // PostDomTree +extern template class DomTreeNodeOnView; +extern template class DominatorTreeOnView; // DomTree +extern template class DominatorTreeOnView; // PostDomTree using MachineDomTree = DomTreeBase; using MachineDomTreeNode = DomTreeNodeBase; diff --git a/llvm/include/llvm/CodeGen/MachineSSAContext.h b/llvm/include/llvm/CodeGen/MachineSSAContext.h index e3b2dc459881..8cf6624e3d20 100644 --- a/llvm/include/llvm/CodeGen/MachineSSAContext.h +++ b/llvm/include/llvm/CodeGen/MachineSSAContext.h @@ -16,6 +16,7 @@ #define LLVM_CODEGEN_MACHINESSACONTEXT_H #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/Support/GenericDomTree.h" #include "llvm/Support/Printable.h" namespace llvm { @@ -24,7 +25,6 @@ class MachineInstr; class MachineFunction; class Register; template class GenericSSAContext; -template class DominatorTreeBase; inline auto successors(const MachineBasicBlock *BB) { return BB->successors(); } inline auto predecessors(const MachineBasicBlock *BB) { diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def b/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def index a31111eb80a4..aa3beea75c75 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def +++ b/llvm/include/llvm/DebugInfo/CodeView/CodeViewTypes.def @@ -53,6 +53,7 @@ TYPE_RECORD(LF_ENUM, 0x1507, Enum) TYPE_RECORD(LF_TYPESERVER2, 0x1515, TypeServer2) TYPE_RECORD(LF_VFTABLE, 0x151d, VFTable) TYPE_RECORD(LF_VTSHAPE, 0x000a, VFTableShape) +TYPE_RECORD(LF_ALIAS, 0x150a, Alias) TYPE_RECORD(LF_BITFIELD, 0x1205, BitField) @@ -181,7 +182,6 @@ CV_TYPE(LF_MANAGED_ST, 0x140f) CV_TYPE(LF_ST_MAX, 0x1500) CV_TYPE(LF_TYPESERVER, 0x1501) CV_TYPE(LF_DIMARRAY, 0x1508) -CV_TYPE(LF_ALIAS, 0x150a) CV_TYPE(LF_DEFARG, 0x150b) CV_TYPE(LF_FRIENDFCN, 0x150c) CV_TYPE(LF_NESTTYPEEX, 0x1512) diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h index 24a4accab845..924b9520038b 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -953,6 +953,19 @@ class EndPrecompRecord : public TypeRecord { uint32_t Signature = 0; }; +// LF_ALIAS +class AliasRecord : public TypeRecord { +public: + AliasRecord() = default; + explicit AliasRecord(TypeRecordKind Kind) : TypeRecord(Kind) {} + AliasRecord(TypeIndex UnderlyingType, StringRef Name) + : TypeRecord(TypeRecordKind::Alias), UnderlyingType(UnderlyingType), + Name(Name) {} + + TypeIndex UnderlyingType; + StringRef Name; +}; + } // end namespace codeview } // end namespace llvm diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h b/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h index 834cd96b77b4..582e613c30a2 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h @@ -16,6 +16,7 @@ #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h" #include "llvm/DebugInfo/PDB/Native/LinePrinter.h" #include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Error.h" @@ -32,7 +33,6 @@ namespace pdb { class InputFile; class LinePrinter; class PDBFile; -class NativeSession; class SymbolGroupIterator; class SymbolGroup; diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h index 0b9ac18d0a92..ca36732e4e2e 100644 --- a/llvm/include/llvm/IR/ConstantRange.h +++ b/llvm/include/llvm/IR/ConstantRange.h @@ -526,6 +526,10 @@ class [[nodiscard]] ConstantRange { /// \p IntMinIsPoison is false. ConstantRange abs(bool IntMinIsPoison = false) const; + /// Calculate ctlz range. If \p ZeroIsPoison is set, the range is computed + /// ignoring a possible zero value contained in the input range. + ConstantRange ctlz(bool ZeroIsPoison = false) const; + /// Represents whether an operation on the given constant range is known to /// always or never overflow. enum class OverflowResult { diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h index c2d080bc2004..52b97b2bc4ea 100644 --- a/llvm/include/llvm/IR/Dominators.h +++ b/llvm/include/llvm/IR/Dominators.h @@ -47,50 +47,57 @@ class Value; class raw_ostream; template struct GraphTraits; -extern template class DomTreeNodeBase; -extern template class DominatorTreeBase; // DomTree -extern template class DominatorTreeBase; // PostDomTree +extern template class DomTreeNodeOnView; +extern template class DominatorTreeOnView; // DomTree +extern template class DominatorTreeOnView; // PostDomTree extern template class cfg::Update; namespace DomTreeBuilder { -using BBDomTree = DomTreeBase; -using BBPostDomTree = PostDomTreeBase; +template class View> +using BBDomTreeOnView = DomTreeOnView; + +using BBDomTree = DomTreeOnView; + +template class View> +using BBPostDomTreeOnView = PostDomTreeOnView; + +using BBPostDomTree = PostDomTreeOnView; using BBUpdates = ArrayRef>; -using BBDomTreeGraphDiff = GraphDiff; -using BBPostDomTreeGraphDiff = GraphDiff; - -extern template void Calculate(BBDomTree &DT); -extern template void CalculateWithUpdates(BBDomTree &DT, - BBUpdates U); - -extern template void Calculate(BBPostDomTree &DT); - -extern template void InsertEdge(BBDomTree &DT, BasicBlock *From, - BasicBlock *To); -extern template void InsertEdge(BBPostDomTree &DT, - BasicBlock *From, - BasicBlock *To); - -extern template void DeleteEdge(BBDomTree &DT, BasicBlock *From, - BasicBlock *To); -extern template void DeleteEdge(BBPostDomTree &DT, - BasicBlock *From, - BasicBlock *To); - -extern template void ApplyUpdates(BBDomTree &DT, - BBDomTreeGraphDiff &, - BBDomTreeGraphDiff *); -extern template void ApplyUpdates(BBPostDomTree &DT, - BBPostDomTreeGraphDiff &, - BBPostDomTreeGraphDiff *); - -extern template bool Verify(const BBDomTree &DT, - BBDomTree::VerificationLevel VL); -extern template bool Verify(const BBPostDomTree &DT, - BBPostDomTree::VerificationLevel VL); +using BBDomTreeGraphDiff = GraphDiff; +using BBPostDomTreeGraphDiff = GraphDiff; + +extern template void Calculate(BBDomTree &DT); +extern template void +CalculateWithUpdates(BBDomTree &DT, BBUpdates U); + +extern template void Calculate(BBPostDomTree &DT); + +extern template void +InsertEdge(BBDomTree &DT, BasicBlock *From, BasicBlock *To); +extern template void +InsertEdge(BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); + +extern template void +DeleteEdge(BBDomTree &DT, BasicBlock *From, BasicBlock *To); +extern template void +DeleteEdge(BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); + +extern template void +ApplyUpdates(BBDomTree &DT, + BBDomTreeGraphDiff &, + BBDomTreeGraphDiff *); +extern template void +ApplyUpdates(BBPostDomTree &DT, + BBPostDomTreeGraphDiff &, + BBPostDomTreeGraphDiff *); + +extern template bool +Verify(const BBDomTree &DT, BBDomTree::VerificationLevel VL); +extern template bool +Verify(const BBPostDomTree &DT, BBPostDomTree::VerificationLevel VL); } // namespace DomTreeBuilder using DomTreeNode = DomTreeNodeBase; diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h index 41c11d26aa45..1dd94b93432d 100644 --- a/llvm/include/llvm/IR/LegacyPassManagers.h +++ b/llvm/include/llvm/IR/LegacyPassManagers.h @@ -498,6 +498,8 @@ class FPPassManager : public ModulePass, public PMDataManager { StringRef getPassName() const override { return "Function Pass Manager"; } + bool isSingleTask() const override { return true; } + FunctionPass *getContainedPass(unsigned N) { assert ( N < PassVector.size() && "Pass number out of range!"); FunctionPass *FP = static_cast(PassVector[N]); diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index e5236523a522..4e249b16eeb6 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -147,7 +147,7 @@ struct alignas(8) GlobalValueSummaryInfo { StringRef Name; } U; - GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {} + inline GlobalValueSummaryInfo(bool HaveGVs); /// List of global value summary structures for a particular value held /// in the GlobalValueMap. Requires a vector in the case of multiple @@ -519,6 +519,8 @@ class GlobalValueSummary { friend class ModuleSummaryIndex; }; +GlobalValueSummaryInfo::GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {} + /// Alias summary information. class AliasSummary : public GlobalValueSummary { ValueInfo AliaseeValueInfo; diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h index 21a0af64da4e..6712e7a7fad1 100644 --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -46,6 +46,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/PassManagerInternal.h" +#include "llvm/Support/Progress.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/TypeName.h" #include @@ -60,6 +61,11 @@ namespace llvm { +template +concept HasName = requires(const T &Object) { + { Object.getName() } -> std::same_as; +}; + /// A special type used by analysis passes to provide an address that /// identifies that particular analysis pass type. /// @@ -507,7 +513,15 @@ class PassManager : public PassInfoMixin< detail::getAnalysisResult( AM, IR, std::tuple(ExtraArgs...)); + std::string Description = "LLVM passes"; + if constexpr (HasName) { + Description += " on " + IR.getName().str(); + } + + Task T(Passes.size(), Description); + for (auto &Pass : Passes) { + T.advance(Pass->name()); // Check the PassInstrumentation's BeforePass callbacks before running the // pass, skip its execution completely if asked to (callback returns // false). diff --git a/llvm/include/llvm/IR/SSAContext.h b/llvm/include/llvm/IR/SSAContext.h index 7551adff1e12..fe5e4885b12a 100644 --- a/llvm/include/llvm/IR/SSAContext.h +++ b/llvm/include/llvm/IR/SSAContext.h @@ -18,6 +18,7 @@ #include "llvm/ADT/GenericSSAContext.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/ModuleSlotTracker.h" +#include "llvm/Support/GenericDomTree.h" #include "llvm/Support/Printable.h" #include @@ -28,7 +29,6 @@ class Function; class Instruction; class Value; template class SmallVectorImpl; -template class DominatorTreeBase; inline auto instrs(const BasicBlock &BB) { return llvm::make_range(BB.begin(), BB.end()); diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h index 56e7c8580b4e..c9e2c3118100 100644 --- a/llvm/include/llvm/Object/MachO.h +++ b/llvm/include/llvm/Object/MachO.h @@ -173,6 +173,7 @@ class MachORebaseEntry { int32_t segmentIndex() const; uint64_t segmentOffset() const; StringRef typeName() const; + MachO::RebaseType type() const; StringRef segmentName() const; StringRef sectionName() const; uint64_t address() const; @@ -196,7 +197,7 @@ class MachORebaseEntry { int32_t SegmentIndex = -1; uint64_t RemainingLoopCount = 0; uint64_t AdvanceAmount = 0; - uint8_t RebaseType = 0; + MachO::RebaseType RebaseType = MachO::REBASE_TYPE_INVALID; uint8_t PointerSize; bool Done = false; }; @@ -219,6 +220,7 @@ class MachOBindEntry { int32_t segmentIndex() const; uint64_t segmentOffset() const; StringRef typeName() const; + MachO::BindType type() const; StringRef symbolName() const; uint32_t flags() const; int64_t addend() const; @@ -253,7 +255,7 @@ class MachOBindEntry { int64_t Addend = 0; uint64_t RemainingLoopCount = 0; uint64_t AdvanceAmount = 0; - uint8_t BindType = 0; + MachO::BindType BindType = MachO::BIND_TYPE_INVALID; uint8_t PointerSize; Kind TableKind; bool Done = false; diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h index 6445e16ab68f..5e3a87d2f9d9 100644 --- a/llvm/include/llvm/Pass.h +++ b/llvm/include/llvm/Pass.h @@ -106,6 +106,8 @@ class Pass { /// Registration templates, but can be overloaded directly. virtual StringRef getPassName() const; + virtual bool isSingleTask() const; + /// getPassID - Return the PassID number that corresponds to this pass. AnalysisID getPassID() const { return PassID; diff --git a/llvm/include/llvm/Support/CFGDiff.h b/llvm/include/llvm/Support/CFGDiff.h index c90b9aca78b5..b0a50690d228 100644 --- a/llvm/include/llvm/Support/CFGDiff.h +++ b/llvm/include/llvm/Support/CFGDiff.h @@ -132,11 +132,12 @@ template class GraphDiff { } using VectRet = SmallVector; - template VectRet getChildren(NodePtr N) const { + template class View> VectRet getChildren(NodePtr N) const { using DirectedNodeT = - std::conditional_t, NodePtr>; - auto R = children(N); - VectRet Res = VectRet(detail::reverse_if(R)); + std::conditional_t>, View>; + VectRet Res = VectRet(children(N)); + if (not InverseEdge) + std::reverse(Res.begin(), Res.end()); // Remove nullptr children for clang. llvm::erase_value(Res, nullptr); diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h index c3ba3274d6ed..e56be721a6aa 100644 --- a/llvm/include/llvm/Support/Compression.h +++ b/llvm/include/llvm/Support/Compression.h @@ -61,6 +61,9 @@ constexpr int BestSizeCompression = 12; bool isAvailable(); +// Will return true if the buffer begins with the zstd magic sequence +bool isZstd(ArrayRef Input); + void compress(ArrayRef Input, SmallVectorImpl &CompressedBuffer, int Level = DefaultCompression); @@ -71,6 +74,12 @@ Error decompress(ArrayRef Input, uint8_t *Output, Error decompress(ArrayRef Input, SmallVectorImpl &Output, size_t UncompressedSize); +// This version of decompress does not require knowing the uncompressed size +// ahead of time, this makes it more flexible than the two versions above but +// also more dangerous since it makes the program more susceptible to running +// out of memory. +Error decompress(ArrayRef Input, SmallVectorImpl &Output); + } // End of namespace zstd enum class Format { diff --git a/llvm/include/llvm/Support/DebugInfoPreservation.h b/llvm/include/llvm/Support/DebugInfoPreservation.h new file mode 100644 index 000000000000..fbf82ab40c63 --- /dev/null +++ b/llvm/include/llvm/Support/DebugInfoPreservation.h @@ -0,0 +1,28 @@ +//===- llvm/Support/CommandLine.h - Command line handler --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an extra command line option that changes debug +// information preservation logic accross multiple transformation utilities. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_DEBUGINFOPRESERVATION_H +#define LLVM_SUPPORT_DEBUGINFOPRESERVATION_H + +#include "llvm/Support/CommandLine.h" + +namespace llvm { + +// This enables stricter debug information preservation. +// In particular, when this is set, debug information should never be dropped, +// even if it leads to worse debugger (gdb, lldb, etc) experience. +extern llvm::cl::opt EnableStrictDebugInformationPreservationStyle; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_DEBUGINFOPRESERVATION_H diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h index 8a984db5e681..714e1de198b2 100644 --- a/llvm/include/llvm/Support/Error.h +++ b/llvm/include/llvm/Support/Error.h @@ -746,12 +746,10 @@ inline void cantFail(Error Err, const char *Msg = nullptr) { if (Err) { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; -#ifndef NDEBUG std::string Str; raw_string_ostream OS(Str); OS << Msg << "\n" << Err; Msg = OS.str().c_str(); -#endif llvm_unreachable(Msg); } } @@ -776,13 +774,11 @@ T cantFail(Expected ValOrErr, const char *Msg = nullptr) { else { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; -#ifndef NDEBUG std::string Str; raw_string_ostream OS(Str); auto E = ValOrErr.takeError(); OS << Msg << "\n" << E; Msg = OS.str().c_str(); -#endif llvm_unreachable(Msg); } } @@ -807,13 +803,11 @@ T& cantFail(Expected ValOrErr, const char *Msg = nullptr) { else { if (!Msg) Msg = "Failure value returned from cantFail wrapped call"; -#ifndef NDEBUG std::string Str; raw_string_ostream OS(Str); auto E = ValOrErr.takeError(); OS << Msg << "\n" << E; Msg = OS.str().c_str(); -#endif llvm_unreachable(Msg); } } @@ -1355,9 +1349,7 @@ Error createFileError(const Twine &F, ErrorSuccess) = delete; class ExitOnError { public: /// Create an error on exit helper. - ExitOnError(std::string Banner = "", int DefaultErrorExitCode = 1) - : Banner(std::move(Banner)), - GetExitCode([=](const Error &) { return DefaultErrorExitCode; }) {} + ExitOnError(std::string Banner = "", int DefaultErrorExitCode = 1); /// Set the banner string for any errors caught by operator(). void setBanner(std::string Banner) { this->Banner = std::move(Banner); } diff --git a/llvm/include/llvm/Support/FormatAdapters.h b/llvm/include/llvm/Support/FormatAdapters.h index 495205d11748..d84e5f8f5f40 100644 --- a/llvm/include/llvm/Support/FormatAdapters.h +++ b/llvm/include/llvm/Support/FormatAdapters.h @@ -75,7 +75,7 @@ class ErrorAdapter : public FormatAdapter { public: ErrorAdapter(Error &&Item) : FormatAdapter(std::move(Item)) {} ErrorAdapter(ErrorAdapter &&) = default; - ~ErrorAdapter() { consumeError(std::move(Item)); } + ~ErrorAdapter() override { consumeError(std::move(Item)); } void format(llvm::raw_ostream &Stream, StringRef Style) override { Stream << Item; } diff --git a/llvm/include/llvm/Support/GenericDomTree.h b/llvm/include/llvm/Support/GenericDomTree.h index 1e5c0ae231d2..9389150adab1 100644 --- a/llvm/include/llvm/Support/GenericDomTree.h +++ b/llvm/include/llvm/Support/GenericDomTree.h @@ -42,44 +42,51 @@ namespace llvm { +template +using DTIdentityView = X; + +template class View> +class DominatorTreeOnView; + template -class DominatorTreeBase; +using DominatorTreeBase = DominatorTreeOnView; namespace DomTreeBuilder { -template -struct SemiNCAInfo; +template class View> +struct SemiNCAInfoOnView; } // namespace DomTreeBuilder /// Base class for the actual dominator tree node. -template class DomTreeNodeBase { +template class View> +class DomTreeNodeOnView { friend class PostDominatorTree; - friend class DominatorTreeBase; - friend class DominatorTreeBase; - friend struct DomTreeBuilder::SemiNCAInfo>; - friend struct DomTreeBuilder::SemiNCAInfo>; + friend class DominatorTreeOnView; + friend class DominatorTreeOnView; + friend struct DomTreeBuilder::SemiNCAInfoOnView; + friend struct DomTreeBuilder::SemiNCAInfoOnView; NodeT *TheBB; - DomTreeNodeBase *IDom; + DomTreeNodeOnView *IDom; unsigned Level; - SmallVector Children; + SmallVector Children; mutable unsigned DFSNumIn = ~0; mutable unsigned DFSNumOut = ~0; public: - DomTreeNodeBase(NodeT *BB, DomTreeNodeBase *iDom) + DomTreeNodeOnView(NodeT *BB, DomTreeNodeOnView *iDom) : TheBB(BB), IDom(iDom), Level(IDom ? IDom->Level + 1 : 0) {} - using iterator = typename SmallVector::iterator; + using iterator = typename SmallVector::iterator; using const_iterator = - typename SmallVector::const_iterator; + typename SmallVector::const_iterator; iterator begin() { return Children.begin(); } iterator end() { return Children.end(); } const_iterator begin() const { return Children.begin(); } const_iterator end() const { return Children.end(); } - DomTreeNodeBase *const &back() const { return Children.back(); } - DomTreeNodeBase *&back() { return Children.back(); } + DomTreeNodeOnView *const &back() const { return Children.back(); } + DomTreeNodeOnView *&back() { return Children.back(); } iterator_range children() { return make_range(begin(), end()); } iterator_range children() const { @@ -87,11 +94,11 @@ template class DomTreeNodeBase { } NodeT *getBlock() const { return TheBB; } - DomTreeNodeBase *getIDom() const { return IDom; } + DomTreeNodeOnView *getIDom() const { return IDom; } unsigned getLevel() const { return Level; } - std::unique_ptr addChild( - std::unique_ptr C) { + std::unique_ptr addChild( + std::unique_ptr C) { Children.push_back(C.get()); return C; } @@ -101,19 +108,19 @@ template class DomTreeNodeBase { void clearAllChildren() { Children.clear(); } - bool compare(const DomTreeNodeBase *Other) const { + bool compare(const DomTreeNodeOnView *Other) const { if (getNumChildren() != Other->getNumChildren()) return true; if (Level != Other->Level) return true; SmallPtrSet OtherChildren; - for (const DomTreeNodeBase *I : *Other) { + for (const DomTreeNodeOnView *I : *Other) { const NodeT *Nd = I->getBlock(); OtherChildren.insert(Nd); } - for (const DomTreeNodeBase *I : *this) { + for (const DomTreeNodeOnView *I : *this) { const NodeT *N = I->getBlock(); if (OtherChildren.count(N) == 0) return true; @@ -121,7 +128,7 @@ template class DomTreeNodeBase { return false; } - void setIDom(DomTreeNodeBase *NewIDom) { + void setIDom(DomTreeNodeOnView *NewIDom) { assert(IDom && "No immediate dominator?"); if (IDom == NewIDom) return; @@ -147,7 +154,7 @@ template class DomTreeNodeBase { private: // Return true if this node is dominated by other. Use this only if DFS info // is valid. - bool DominatedBy(const DomTreeNodeBase *other) const { + bool DominatedBy(const DomTreeNodeOnView *other) const { return this->DFSNumIn >= other->DFSNumIn && this->DFSNumOut <= other->DFSNumOut; } @@ -156,13 +163,13 @@ template class DomTreeNodeBase { assert(IDom); if (Level == IDom->Level + 1) return; - SmallVector WorkStack = {this}; + SmallVector WorkStack = {this}; while (!WorkStack.empty()) { - DomTreeNodeBase *Current = WorkStack.pop_back_val(); + DomTreeNodeOnView *Current = WorkStack.pop_back_val(); Current->Level = Current->IDom->Level + 1; - for (DomTreeNodeBase *C : *Current) { + for (DomTreeNodeOnView *C : *Current) { assert(C->IDom); if (C->Level != C->IDom->Level + 1) WorkStack.push_back(C); } @@ -171,7 +178,10 @@ template class DomTreeNodeBase { }; template -raw_ostream &operator<<(raw_ostream &O, const DomTreeNodeBase *Node) { +using DomTreeNodeBase = DomTreeNodeOnView; + +template class View> +raw_ostream &operator<<(raw_ostream &O, const DomTreeNodeOnView *Node) { if (Node->getBlock()) Node->getBlock()->printAsOperand(O, false); else @@ -183,11 +193,11 @@ raw_ostream &operator<<(raw_ostream &O, const DomTreeNodeBase *Node) { return O; } -template -void PrintDomTree(const DomTreeNodeBase *N, raw_ostream &O, +template class View> +void PrintDomTree(const DomTreeNodeOnView *N, raw_ostream &O, unsigned Lev) { O.indent(2 * Lev) << "[" << Lev << "] " << N; - for (typename DomTreeNodeBase::const_iterator I = N->begin(), + for (typename DomTreeNodeOnView::const_iterator I = N->begin(), E = N->end(); I != E; ++I) PrintDomTree(*I, O, Lev + 1); @@ -195,30 +205,31 @@ void PrintDomTree(const DomTreeNodeBase *N, raw_ostream &O, namespace DomTreeBuilder { // The routines below are provided in a separate header but referenced here. -template -void Calculate(DomTreeT &DT); - -template -void CalculateWithUpdates(DomTreeT &DT, - ArrayRef Updates); - -template -void InsertEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To); - -template -void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To); - -template -void ApplyUpdates(DomTreeT &DT, - GraphDiff &PreViewCFG, - GraphDiff *PostViewCFG); - -template -bool Verify(const DomTreeT &DT, typename DomTreeT::VerificationLevel VL); +template class View> +void Calculate(DominatorTreeOnView &DT); + +template class View> +void CalculateWithUpdates(DominatorTreeOnView &DT, + ArrayRef::UpdateType> Updates); + +template class View> +void InsertEdge(DominatorTreeOnView &DT, + typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To); + +template class View> +void DeleteEdge(DominatorTreeOnView &DT, + typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To); + +template class View> +void ApplyUpdates(DominatorTreeOnView &DT, + GraphDiff::NodePtr, IsPostDom> &PreViewCFG, + GraphDiff::NodePtr, IsPostDom> *PostViewCFG); + +template class View> +bool Verify(const DominatorTreeOnView &DT, + typename DominatorTreeOnView::VerificationLevel VL); } // namespace DomTreeBuilder /// Default DomTreeNode traits for NodeT. The default implementation assume a @@ -239,11 +250,11 @@ template struct DomTreeNodeTraits { /// /// This class is a generic template over graph nodes. It is instantiated for /// various graphs in the LLVM IR or in the code generator. -template -class DominatorTreeBase { +template class View> +class DominatorTreeOnView { public: static_assert(std::is_pointer::NodeRef>::value, - "Currently DominatorTreeBase supports only pointer nodes"); + "Currently DominatorTreeOnView supports only pointer nodes"); using NodeTrait = DomTreeNodeTraits; using NodeType = typename NodeTrait::NodeType; using NodePtr = typename NodeTrait::NodePtr; @@ -265,20 +276,21 @@ class DominatorTreeBase { SmallVector Roots; using DomTreeNodeMapType = - DenseMap>>; + DenseMap>>; DomTreeNodeMapType DomTreeNodes; - DomTreeNodeBase *RootNode = nullptr; + DomTreeNodeOnView *RootNode = nullptr; ParentPtr Parent = nullptr; mutable bool DFSInfoValid = false; mutable unsigned int SlowQueries = 0; - friend struct DomTreeBuilder::SemiNCAInfo; + friend struct DomTreeBuilder::SemiNCAInfoOnView; + friend struct DomTreeBuilder::SemiNCAInfoOnView; public: - DominatorTreeBase() = default; + DominatorTreeOnView() = default; - DominatorTreeBase(DominatorTreeBase &&Arg) + DominatorTreeOnView(DominatorTreeOnView &&Arg) : Roots(std::move(Arg.Roots)), DomTreeNodes(std::move(Arg.DomTreeNodes)), RootNode(Arg.RootNode), @@ -288,7 +300,7 @@ class DominatorTreeBase { Arg.wipe(); } - DominatorTreeBase &operator=(DominatorTreeBase &&RHS) { + DominatorTreeOnView &operator=(DominatorTreeOnView &&RHS) { Roots = std::move(RHS.Roots); DomTreeNodes = std::move(RHS.DomTreeNodes); RootNode = RHS.RootNode; @@ -299,8 +311,8 @@ class DominatorTreeBase { return *this; } - DominatorTreeBase(const DominatorTreeBase &) = delete; - DominatorTreeBase &operator=(const DominatorTreeBase &) = delete; + DominatorTreeOnView(const DominatorTreeOnView &) = delete; + DominatorTreeOnView &operator=(const DominatorTreeOnView &) = delete; /// Iteration over roots. /// @@ -330,7 +342,7 @@ class DominatorTreeBase { /// compare - Return false if the other dominator tree base matches this /// dominator tree base. Otherwise return true. - bool compare(const DominatorTreeBase &Other) const { + bool compare(const DominatorTreeOnView &Other) const { if (Parent != Other.Parent) return true; if (Roots.size() != Other.Roots.size()) @@ -350,8 +362,8 @@ class DominatorTreeBase { if (OI == OtherDomTreeNodes.end()) return true; - DomTreeNodeBase &MyNd = *DomTreeNode.second; - DomTreeNodeBase &OtherNd = *OI->second; + DomTreeNodeOnView &MyNd = *DomTreeNode.second; + DomTreeNodeOnView &OtherNd = *OI->second; if (MyNd.compare(&OtherNd)) return true; @@ -364,7 +376,7 @@ class DominatorTreeBase { /// block. This is the same as using operator[] on this class. The result /// may (but is not required to) be null for a forward (backwards) /// statically unreachable block. - DomTreeNodeBase *getNode(const NodeT *BB) const { + DomTreeNodeOnView *getNode(const NodeT *BB) const { auto I = DomTreeNodes.find(BB); if (I != DomTreeNodes.end()) return I->second.get(); @@ -372,7 +384,7 @@ class DominatorTreeBase { } /// See getNode. - DomTreeNodeBase *operator[](const NodeT *BB) const { + DomTreeNodeOnView *operator[](const NodeT *BB) const { return getNode(BB); } @@ -383,20 +395,20 @@ class DominatorTreeBase { /// post-dominance information must be capable of dealing with this /// possibility. /// - DomTreeNodeBase *getRootNode() { return RootNode; } - const DomTreeNodeBase *getRootNode() const { return RootNode; } + DomTreeNodeOnView *getRootNode() { return RootNode; } + const DomTreeNodeOnView *getRootNode() const { return RootNode; } /// Get all nodes dominated by R, including R itself. void getDescendants(NodeT *R, SmallVectorImpl &Result) const { Result.clear(); - const DomTreeNodeBase *RN = getNode(R); + const DomTreeNodeOnView *RN = getNode(R); if (!RN) return; // If R is unreachable, it will not be present in the DOM tree. - SmallVector *, 8> WL; + SmallVector *, 8> WL; WL.push_back(RN); while (!WL.empty()) { - const DomTreeNodeBase *N = WL.pop_back_val(); + const DomTreeNodeOnView *N = WL.pop_back_val(); Result.push_back(N->getBlock()); WL.append(N->begin(), N->end()); } @@ -405,8 +417,8 @@ class DominatorTreeBase { /// properlyDominates - Returns true iff A dominates B and A != B. /// Note that this is not a constant time operation! /// - bool properlyDominates(const DomTreeNodeBase *A, - const DomTreeNodeBase *B) const { + bool properlyDominates(const DomTreeNodeOnView *A, + const DomTreeNodeOnView *B) const { if (!A || !B) return false; if (A == B) @@ -424,13 +436,13 @@ class DominatorTreeBase { return isReachableFromEntry(getNode(const_cast(A))); } - bool isReachableFromEntry(const DomTreeNodeBase *A) const { return A; } + bool isReachableFromEntry(const DomTreeNodeOnView *A) const { return A; } /// dominates - Returns true iff A dominates B. Note that this is not a /// constant time operation! /// - bool dominates(const DomTreeNodeBase *A, - const DomTreeNodeBase *B) const { + bool dominates(const DomTreeNodeOnView *A, + const DomTreeNodeOnView *B) const { // A node trivially dominates itself. if (B == A) return true; @@ -495,8 +507,8 @@ class DominatorTreeBase { return &Entry; } - DomTreeNodeBase *NodeA = getNode(A); - DomTreeNodeBase *NodeB = getNode(B); + DomTreeNodeOnView *NodeA = getNode(A); + DomTreeNodeOnView *NodeB = getNode(B); assert(NodeA && "A must be in the tree"); assert(NodeB && "B must be in the tree"); @@ -519,7 +531,7 @@ class DominatorTreeBase { const_cast(B)); } - bool isVirtualRoot(const DomTreeNodeBase *A) const { + bool isVirtualRoot(const DomTreeNodeOnView *A) const { return isPostDominator() && !A->getBlock(); } @@ -561,7 +573,7 @@ class DominatorTreeBase { void applyUpdates(ArrayRef Updates) { GraphDiff PreViewCFG( Updates, /*ReverseApplyUpdates=*/true); - DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, nullptr); + DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, nullptr); } /// \param Updates An ordered sequence of updates to perform. The current CFG @@ -573,7 +585,7 @@ class DominatorTreeBase { ArrayRef PostViewUpdates) { if (Updates.empty()) { GraphDiff PostViewCFG(PostViewUpdates); - DomTreeBuilder::ApplyUpdates(*this, PostViewCFG, &PostViewCFG); + DomTreeBuilder::ApplyUpdates(*this, PostViewCFG, &PostViewCFG); } else { // PreViewCFG needs to merge Updates and PostViewCFG. The updates in // Updates need to be reversed, and match the direction in PostViewCFG. @@ -585,7 +597,7 @@ class DominatorTreeBase { GraphDiff PreViewCFG(AllUpdates, /*ReverseApplyUpdates=*/true); GraphDiff PostViewCFG(PostViewUpdates); - DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, &PostViewCFG); + DomTreeBuilder::ApplyUpdates(*this, PreViewCFG, &PostViewCFG); } } @@ -633,9 +645,9 @@ class DominatorTreeBase { /// \param DomBB CFG node that is dominator for BB. /// \returns New dominator tree node that represents new CFG node. /// - DomTreeNodeBase *addNewBlock(NodeT *BB, NodeT *DomBB) { + DomTreeNodeOnView *addNewBlock(NodeT *BB, NodeT *DomBB) { assert(getNode(BB) == nullptr && "Block already in dominator tree!"); - DomTreeNodeBase *IDomNode = getNode(DomBB); + DomTreeNodeOnView *IDomNode = getNode(DomBB); assert(IDomNode && "Not immediate dominator specified for block!"); DFSInfoValid = false; return createChild(BB, IDomNode); @@ -646,12 +658,12 @@ class DominatorTreeBase { /// \param BB New node in CFG. /// \returns New dominator tree node that represents new CFG node. /// - DomTreeNodeBase *setNewRoot(NodeT *BB) { + DomTreeNodeOnView *setNewRoot(NodeT *BB) { assert(getNode(BB) == nullptr && "Block already in dominator tree!"); assert(!this->isPostDominator() && "Cannot change root of post-dominator tree"); DFSInfoValid = false; - DomTreeNodeBase *NewNode = createNode(BB); + DomTreeNodeOnView *NewNode = createNode(BB); if (Roots.empty()) { addRoot(BB); } else { @@ -669,8 +681,8 @@ class DominatorTreeBase { /// changeImmediateDominator - This method is used to update the dominator /// tree information when a node's immediate dominator changes. /// - void changeImmediateDominator(DomTreeNodeBase *N, - DomTreeNodeBase *NewIDom) { + void changeImmediateDominator(DomTreeNodeOnView *N, + DomTreeNodeOnView *NewIDom) { assert(N && NewIDom && "Cannot change null node pointers!"); DFSInfoValid = false; N->setIDom(NewIDom); @@ -684,14 +696,14 @@ class DominatorTreeBase { /// dominate any other blocks. Removes node from its immediate dominator's /// children list. Deletes dominator node associated with basic block BB. void eraseNode(NodeT *BB) { - DomTreeNodeBase *Node = getNode(BB); + DomTreeNodeOnView *Node = getNode(BB); assert(Node && "Removing node that isn't in dominator tree."); assert(Node->isLeaf() && "Node is not a leaf node."); DFSInfoValid = false; // Remove node from immediate dominator's children list. - DomTreeNodeBase *IDom = Node->getIDom(); + DomTreeNodeOnView *IDom = Node->getIDom(); if (IDom) { const auto I = find(IDom->Children, Node); assert(I != IDom->Children.end() && @@ -716,9 +728,9 @@ class DominatorTreeBase { /// tree to reflect this change. void splitBlock(NodeT *NewBB) { if (IsPostDominator) - Split>(NewBB); + Split>>(NewBB); else - Split(NewBB); + Split>(NewBB); } /// print - Convert to human readable form @@ -752,11 +764,11 @@ class DominatorTreeBase { return; } - SmallVector *, - typename DomTreeNodeBase::const_iterator>, + SmallVector *, + typename DomTreeNodeOnView::const_iterator>, 32> WorkStack; - const DomTreeNodeBase *ThisRoot = getRootNode(); + const DomTreeNodeOnView *ThisRoot = getRootNode(); assert((!Parent || ThisRoot) && "Empty constructed DomTree"); if (!ThisRoot) return; @@ -769,7 +781,7 @@ class DominatorTreeBase { ThisRoot->DFSNumIn = DFSNum++; while (!WorkStack.empty()) { - const DomTreeNodeBase *Node = WorkStack.back().first; + const DomTreeNodeOnView *Node = WorkStack.back().first; const auto ChildIt = WorkStack.back().second; // If we visited all of the children of this node, "recurse" back up the @@ -779,7 +791,7 @@ class DominatorTreeBase { WorkStack.pop_back(); } else { // Otherwise, recursively visit this child. - const DomTreeNodeBase *Child = *ChildIt; + const DomTreeNodeOnView *Child = *ChildIt; ++WorkStack.back().second; WorkStack.push_back({Child, Child->begin()}); @@ -832,15 +844,16 @@ class DominatorTreeBase { protected: void addRoot(NodeT *BB) { this->Roots.push_back(BB); } - DomTreeNodeBase *createChild(NodeT *BB, DomTreeNodeBase *IDom) { + DomTreeNodeOnView * + createChild(NodeT *BB, DomTreeNodeOnView *IDom) { return (DomTreeNodes[BB] = IDom->addChild( - std::make_unique>(BB, IDom))) + std::make_unique>(BB, IDom))) .get(); } - DomTreeNodeBase *createNode(NodeT *BB) { + DomTreeNodeOnView *createNode(NodeT *BB) { return (DomTreeNodes[BB] = - std::make_unique>(BB, nullptr)) + std::make_unique>(BB, nullptr)) .get(); } @@ -889,25 +902,25 @@ class DominatorTreeBase { } // Create the new dominator tree node... and set the idom of NewBB. - DomTreeNodeBase *NewBBNode = addNewBlock(NewBB, NewBBIDom); + DomTreeNodeOnView *NewBBNode = addNewBlock(NewBB, NewBBIDom); // If NewBB strictly dominates other blocks, then it is now the immediate // dominator of NewBBSucc. Update the dominator tree as appropriate. if (NewBBDominatesNewBBSucc) { - DomTreeNodeBase *NewBBSuccNode = getNode(NewBBSucc); + DomTreeNodeOnView *NewBBSuccNode = getNode(NewBBSucc); changeImmediateDominator(NewBBSuccNode, NewBBNode); } } private: - bool dominatedBySlowTreeWalk(const DomTreeNodeBase *A, - const DomTreeNodeBase *B) const { + bool dominatedBySlowTreeWalk(const DomTreeNodeOnView *A, + const DomTreeNodeOnView *B) const { assert(A != B); assert(isReachableFromEntry(B)); assert(isReachableFromEntry(A)); const unsigned ALevel = A->getLevel(); - const DomTreeNodeBase *IDom; + const DomTreeNodeOnView *IDom; // Don't walk nodes above A's subtree. When we reach A's level, we must // either find A or be in some other subtree not dominated by A. @@ -928,16 +941,22 @@ class DominatorTreeBase { } }; +template class View> +using DomTreeOnView = DominatorTreeOnView; + template -using DomTreeBase = DominatorTreeBase; +using DomTreeBase = DomTreeOnView; + +template class View> +using PostDomTreeOnView = DominatorTreeOnView; template -using PostDomTreeBase = DominatorTreeBase; +using PostDomTreeBase = PostDomTreeOnView; // These two functions are declared out of line as a workaround for building // with old (< r147295) versions of clang because of pr11642. -template -bool DominatorTreeBase::dominates(const NodeT *A, +template class View> +bool DominatorTreeOnView::dominates(const NodeT *A, const NodeT *B) const { if (A == B) return true; @@ -948,8 +967,8 @@ bool DominatorTreeBase::dominates(const NodeT *A, return dominates(getNode(const_cast(A)), getNode(const_cast(B))); } -template -bool DominatorTreeBase::properlyDominates( +template class View> +bool DominatorTreeOnView::properlyDominates( const NodeT *A, const NodeT *B) const { if (A == B) return false; diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h index 2e7716cb0b4b..f48a1b56bb8e 100644 --- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h +++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h @@ -52,13 +52,13 @@ namespace llvm { namespace DomTreeBuilder { -template -struct SemiNCAInfo { +template class View> +struct SemiNCAInfoOnView { + using DomTreeT = DominatorTreeOnView; using NodePtr = typename DomTreeT::NodePtr; - using NodeT = typename DomTreeT::NodeType; - using TreeNodePtr = DomTreeNodeBase *; + using TreeNode = DomTreeNodeOnView; + using TreeNodePtr = TreeNode*; using RootsT = decltype(DomTreeT::Roots); - static constexpr bool IsPostDom = DomTreeT::IsPostDominator; using GraphDiffT = GraphDiff; // Information record used by Semi-NCA during tree construction. @@ -96,7 +96,7 @@ struct SemiNCAInfo { using BatchUpdatePtr = BatchUpdateInfo *; // If BUI is a nullptr, then there's no batch update in progress. - SemiNCAInfo(BatchUpdatePtr BUI) : BatchUpdates(BUI) {} + SemiNCAInfoOnView(BatchUpdatePtr BUI) : BatchUpdates(BUI) {} void clear() { NumToNode = {nullptr}; // Restore to initial state with a dummy start node. @@ -108,16 +108,18 @@ struct SemiNCAInfo { template static SmallVector getChildren(NodePtr N, BatchUpdatePtr BUI) { if (BUI) - return BUI->PreViewCFG.template getChildren(N); + return BUI->PreViewCFG.template getChildren(N); return getChildren(N); } template static SmallVector getChildren(NodePtr N) { using DirectedNodeT = - std::conditional_t, NodePtr>; - auto R = children(N); - SmallVector Res(detail::reverse_if(R)); + std::conditional_t>, View>; + using VectRet = SmallVector; + VectRet Res = VectRet(children(N)); + if (not Inversed) + std::reverse(Res.begin(), Res.end()); // Remove nullptr children for clang. llvm::erase_value(Res, nullptr); @@ -241,7 +243,7 @@ struct SemiNCAInfo { NodePtr eval(NodePtr V, unsigned LastLinked, SmallVectorImpl &Stack) { InfoRec *VInfo = &NodeToInfo[V]; - if (VInfo->Parent < LastLinked) + if ((VInfo->Parent) < (LastLinked)) return VInfo->Label; // Store ancestors except the last (root of a virtual tree) into a stack. @@ -345,20 +347,11 @@ struct SemiNCAInfo { return GraphTraits::getEntryNode(DT.Parent); } - // Finds all roots without relaying on the set of roots already stored in the - // tree. - // We define roots to be some non-redundant set of the CFG nodes - static RootsT FindRoots(const DomTreeT &DT, BatchUpdatePtr BUI) { - assert(DT.Parent && "Parent pointer is not set"); + // We factor the code of `FindRoots` for postdominators in this static + // function so we can constexpr away it when instantiating it on a dominator. + static RootsT FindRootsPostDom(const DomTreeT &DT, BatchUpdatePtr BUI) { RootsT Roots; - - // For dominators, function entry CFG node is always a tree root node. - if (!IsPostDom) { - Roots.push_back(GetEntryNode(DT)); - return Roots; - } - - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); // PostDominatorTree always has a virtual root. SNCA.addVirtualRoot(); @@ -496,6 +489,24 @@ struct SemiNCAInfo { return Roots; } + // Finds all roots without relaying on the set of roots already stored in the + // tree. + // We define roots to be some non-redundant set of the CFG nodes + static RootsT FindRoots(const DomTreeT &DT, BatchUpdatePtr BUI) { + assert(DT.Parent && "Parent pointer is not set"); + + // We avoid compiling the code handling postdominators, when instantiating a + // dominator. + if constexpr (IsPostDom) { + return FindRootsPostDom(DT, BUI); + } + + // For dominators, function entry CFG node is always a tree root node. + RootsT Roots; + Roots.push_back(GetEntryNode(DT)); + return Roots; + } + // This function only makes sense for postdominators. // We define roots to be some set of CFG nodes where (reverse) DFS walks have // to start in order to visit all the CFG nodes (including the @@ -509,7 +520,7 @@ struct SemiNCAInfo { assert(IsPostDom && "This function is for postdominators only"); LLVM_DEBUG(dbgs() << "Removing redundant roots\n"); - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); for (unsigned i = 0; i < Roots.size(); ++i) { auto &Root = Roots[i]; @@ -570,7 +581,7 @@ struct SemiNCAInfo { } // This is rebuilding the whole tree, not incrementally, but PostViewBUI is // used in case the caller needs a DT update with a CFGView. - SemiNCAInfo SNCA(PostViewBUI); + SemiNCAInfoOnView SNCA(PostViewBUI); // Step #0: Number blocks in depth-first order and initialize variables used // in later stages of the algorithm. @@ -903,7 +914,7 @@ struct SemiNCAInfo { return false; }; - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); SNCA.runDFS(Root, 0, UnreachableDescender, 0); SNCA.runSemiNCA(DT); SNCA.attachNewSubtree(DT, Incoming); @@ -996,7 +1007,7 @@ struct SemiNCAInfo { LLVM_DEBUG(dbgs() << "\tTop of subtree: " << BlockNamePrinter(ToIDomTN) << "\n"); - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); SNCA.runDFS(ToIDom, 0, DescendBelow, 0); LLVM_DEBUG(dbgs() << "\tRunning Semi-NCA\n"); SNCA.runSemiNCA(DT, Level); @@ -1063,7 +1074,7 @@ struct SemiNCAInfo { return false; }; - SemiNCAInfo SNCA(BUI); + SemiNCAInfoOnView SNCA(BUI); unsigned LastDFSNum = SNCA.runDFS(ToTN->getBlock(), 0, DescendAndCollect, 0); @@ -1557,48 +1568,46 @@ struct SemiNCAInfo { } }; -template -void Calculate(DomTreeT &DT) { - SemiNCAInfo::CalculateFromScratch(DT, nullptr); +template class View> +void Calculate(DominatorTreeOnView &DT) { + SemiNCAInfoOnView::CalculateFromScratch(DT, nullptr); } -template -void CalculateWithUpdates(DomTreeT &DT, - ArrayRef Updates) { +template class View> +void CalculateWithUpdates(DominatorTreeOnView &DT, + ArrayRef::UpdateType> Updates) { // FIXME: Updated to use the PreViewCFG and behave the same as until now. // This behavior is however incorrect; this actually needs the PostViewCFG. - GraphDiff PreViewCFG( + GraphDiff::NodePtr, IsPostDom> PreViewCFG( Updates, /*ReverseApplyUpdates=*/true); - typename SemiNCAInfo::BatchUpdateInfo BUI(PreViewCFG); - SemiNCAInfo::CalculateFromScratch(DT, &BUI); + typename SemiNCAInfoOnView::BatchUpdateInfo BUI(PreViewCFG); + SemiNCAInfoOnView::CalculateFromScratch(DT, &BUI); } -template -void InsertEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To) { +template class View> +void InsertEdge(DominatorTreeOnView &DT, typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To) { if (DT.isPostDominator()) std::swap(From, To); - SemiNCAInfo::InsertEdge(DT, nullptr, From, To); + SemiNCAInfoOnView::InsertEdge(DT, nullptr, From, To); } -template -void DeleteEdge(DomTreeT &DT, typename DomTreeT::NodePtr From, - typename DomTreeT::NodePtr To) { +template class View> +void DeleteEdge(DominatorTreeOnView &DT, typename DominatorTreeOnView::NodePtr From, + typename DominatorTreeOnView::NodePtr To) { if (DT.isPostDominator()) std::swap(From, To); - SemiNCAInfo::DeleteEdge(DT, nullptr, From, To); + SemiNCAInfoOnView::DeleteEdge(DT, nullptr, From, To); } -template -void ApplyUpdates(DomTreeT &DT, - GraphDiff &PreViewCFG, - GraphDiff *PostViewCFG) { - SemiNCAInfo::ApplyUpdates(DT, PreViewCFG, PostViewCFG); +template class View> +void ApplyUpdates(DominatorTreeOnView &DT, + GraphDiff::NodePtr, IsPostDom> &PreViewCFG, + GraphDiff::NodePtr, IsPostDom> *PostViewCFG) { + SemiNCAInfoOnView::ApplyUpdates(DT, PreViewCFG, PostViewCFG); } -template -bool Verify(const DomTreeT &DT, typename DomTreeT::VerificationLevel VL) { - SemiNCAInfo SNCA(nullptr); +template class View> +bool Verify(const DominatorTreeOnView &DT, typename DominatorTreeOnView::VerificationLevel VL) { + SemiNCAInfoOnView SNCA(nullptr); // Simplist check is to compare against a new tree. This will also // usefully print the old and new trees, if they are different. @@ -1610,12 +1619,12 @@ bool Verify(const DomTreeT &DT, typename DomTreeT::VerificationLevel VL) { !SNCA.VerifyLevels(DT) || !SNCA.VerifyDFSNumbers(DT)) return false; - // Extra checks depending on VerificationLevel. Up to O(N^3). - if (VL == DomTreeT::VerificationLevel::Basic || - VL == DomTreeT::VerificationLevel::Full) + // Extra checks depending on VerificationLevel. Up to O(N^3) + if (VL == DominatorTreeOnView::VerificationLevel::Basic || + VL == DominatorTreeOnView::VerificationLevel::Full) if (!SNCA.verifyParentProperty(DT)) return false; - if (VL == DomTreeT::VerificationLevel::Full) + if (VL == DominatorTreeOnView::VerificationLevel::Full) if (!SNCA.verifySiblingProperty(DT)) return false; diff --git a/llvm/include/llvm/Support/Progress.h b/llvm/include/llvm/Support/Progress.h new file mode 100644 index 000000000000..4336af97c61d --- /dev/null +++ b/llvm/include/llvm/Support/Progress.h @@ -0,0 +1,255 @@ +//===-- Progress - Generic infrastructure to report progress ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_PROGRESS_H +#define LLVM_SUPPORT_PROGRESS_H + +extern "C" { +#include +} + +#include +#include +#include +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Threading.h" + +namespace llvm { + +class Progress; +class TaskStack; + +/// Track advancement of a task +/// +/// A task can have a name and optionally a set of maximum steps. +class Task { +protected: + TaskStack &Stack; + int64_t StepIndex = -1; + std::optional TotalSteps; + +private: + std::string TaskName; + std::string StepName; + bool SubtaskCreated = false; + bool Completed = false; + bool CurrentStepHasSingleSubtask = false; + +public: + Task(std::optional TotalSteps, const llvm::Twine &TaskName = ""); + + Task(const Task &) = delete; + Task(Task &&) = delete; + + ~Task(); + +public: + llvm::StringRef name() const { return TaskName; } + llvm::StringRef stepName() const { return StepName; } + /// \return -1 if no step (i.e., call to advance) has been initiated, + /// the index of the last initiated step otherwise. + int64_t stepIndex() const { return StepIndex; } + std::optional totalSteps() const { return TotalSteps; } + bool subtaskCreated() const { return SubtaskCreated; } + bool currentStepHasSingleSubtask() const { + return CurrentStepHasSingleSubtask; + } + bool completed() const { return Completed; } + size_t index() const; + + TaskStack &stack() { return Stack; } + const TaskStack &stack() const { return Stack; } + +public: + /// \param NewStepName the name of the new step starting after this advance + /// invocation. + /// \param SingleSubtask whether we can expect 0 or 1 subtask being created + /// during this step. If true, will assert in case multiple subtasks + /// are create during this step. + void advance(const llvm::Twine &NewStepName = "", bool SingleSubtask = false); + + void setSubtaskCreated(); + + void complete(); +}; + +/// Represents the set of tasks started but not completed in an execution thread +class TaskStack { +private: + Progress &Parent; + unsigned SuspendRequests = 0; + +public: + llvm::SmallVector Tasks; + +public: + TaskStack(Progress &Parent) : Parent(Parent) {} + + TaskStack(TaskStack &&) = delete; + TaskStack(const TaskStack &) = delete; + TaskStack &operator=(const TaskStack &) = delete; + TaskStack &operator=(TaskStack &&) = delete; + +public: + void registerTask(Task *T); + void unregisterTask(Task *T); + void advanceTask(Task *T, llvm::StringRef PreviousStepName); + +public: + bool isSuspended() const { return SuspendRequests > 0; } + void resumeTracking() { + assert(isSuspended()); + --SuspendRequests; + } + + void suspendTracking() { ++SuspendRequests; } +}; + +/// Class to monitor progress of a Progress instance +class ProgressListener { +public: + virtual ~ProgressListener() = default; + + /// Invoked to notify the creation of a new task + /// + /// \note \p T will always be the last task of its stack. + /// + /// \note This method could be called by multiple threads in parallel. + virtual void handleNewTask(const Task *T) = 0; + + /// Invoked to notify the completion of a task + /// + /// \note \p T will always be the last task of its stack. + /// + /// \note This method could be called by multiple threads in parallel. + virtual void handleTaskCompleted(const Task *T) = 0; + + /// Invoked to notify of the advancement of a certain task to a new step + /// + /// \note \p T will always be the last task of its stack. + /// + /// \note This method could be called by multiple threads in parallel. + virtual void handleTaskAdvancement(const Task *T, + llvm::StringRef PreviousStepName) = 0; +}; + +/// Class to monitor progress of task staks over multiple threads +class Progress { +private: + struct RegistryEntry { + bool AllThreads = false; + std::unique_ptr Listener; + }; + +private: + uint64_t MainThreadID; + llvm::SmallVector Registry; + +public: + Progress(); + +public: + /// \note call this method from the main thread only. + template + void registerListener(Types &&...Args) { + assert(isMainThread()); + Registry.push_back({T::AllThreads, std::make_unique(Args...)}); + } + +public: + bool isMainThread() const; + +public: + void handleNewTask(const Task *T); + void handleTaskCompleted(const Task *T); + void handleTaskAdvancement(const Task *T, llvm::StringRef PreviousStepName); +}; + +/// The default instance of Progress each Task will use +extern llvm::ManagedStatic ProgressReport; + +/// Similar to Task, but gets a list of objects at construction times that +/// represent elements the task iterates on. +/// Each invocation of advance accepts a similar object: if the passed object is +/// not in the list of expected objects, the monitor of progress will be +/// suspended until the next step. +template , + llvm::SmallPtrSet, + std::set>> +class TaskOnSet : public Task { +private: + Set Predicted; + bool Suspended = false; + +public: + template + TaskOnSet(R &&Elements, const llvm::Twine &TaskName = "") + : Task({}, TaskName) { + unsigned Count = 0; + for (const auto &Element : Elements) { + Predicted.insert(Element); + ++Count; + } + + TotalSteps = Predicted.size(); + + assert(TotalSteps == Count); + } + + ~TaskOnSet() { + if (Suspended) + Stack.resumeTracking(); + } + +public: + void advance(const ValueType &Element, const llvm::Twine &NewStepName = "", + bool SingleSubtask = false) { + + if (Predicted.count(Element) != 0) { + if (Suspended) { + Stack.resumeTracking(); + Suspended = false; + } + + // This is an expected element, proceed + Task::advance(NewStepName, SingleSubtask); + + } else { + if (not Suspended) { + // Ignore all unexpected elements + Stack.suspendTracking(); + Suspended = true; + } + } + } +}; + +template +auto make_task_on_set(R &&Elements, const llvm::Twine &TaskName = "") { + return TaskOnSet().begin())>>( + Elements, TaskName); +} + +template +auto make_task_on_set(TaskStack &Stack, R &&Elements, + const llvm::Twine &TaskName = "") { + return TaskOnSet().begin())>>( + Stack, Elements, TaskName); +} + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h index b91acb576ba5..d21fc5ca7737 100644 --- a/llvm/include/llvm/Support/ScopedPrinter.h +++ b/llvm/include/llvm/Support/ScopedPrinter.h @@ -519,7 +519,13 @@ ScopedPrinter::printHex(StringRef Label, startLine() << Label << ": " << hex(Value) << "\n"; } -struct DelimitedScope; +struct DelimitedScope { + DelimitedScope(ScopedPrinter &W) : W(&W) {} + DelimitedScope() : W(nullptr) {} + virtual ~DelimitedScope() = default; + virtual void setPrinter(ScopedPrinter &W) = 0; + ScopedPrinter *W; +}; class JSONScopedPrinter : public ScopedPrinter { private: @@ -798,14 +804,6 @@ class JSONScopedPrinter : public ScopedPrinter { } }; -struct DelimitedScope { - DelimitedScope(ScopedPrinter &W) : W(&W) {} - DelimitedScope() : W(nullptr) {} - virtual ~DelimitedScope() = default; - virtual void setPrinter(ScopedPrinter &W) = 0; - ScopedPrinter *W; -}; - struct DictScope : DelimitedScope { explicit DictScope() = default; explicit DictScope(ScopedPrinter &W) : DelimitedScope(W) { W.objectBegin(); } @@ -819,7 +817,7 @@ struct DictScope : DelimitedScope { W.objectBegin(); } - ~DictScope() { + ~DictScope() override { if (W) W->objectEnd(); } @@ -838,7 +836,7 @@ struct ListScope : DelimitedScope { W.arrayBegin(); } - ~ListScope() { + ~ListScope() override { if (W) W->arrayEnd(); } diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index ef74f38671b5..482e059ba16e 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -665,7 +665,12 @@ inline bool isNull(StringRef S) { inline bool isBool(StringRef S) { // FIXME: using parseBool is causing multiple tests to fail. return S.equals("true") || S.equals("True") || S.equals("TRUE") || - S.equals("false") || S.equals("False") || S.equals("FALSE"); + S.equals("yes") || S.equals("Yes") || S.equals ("YES") || + S.equals("on") || S.equals("On") || S.equals ("ON") || + S.equals("false") || S.equals("False") || S.equals("FALSE") || + S.equals("no") || S.equals("No") || S.equals("NO") || + S.equals("off") || S.equals("Off") || S.equals("OFF") || + S.equals("y") || S.equals("Y") || S.equals("n") || S.equals("N"); } // 5.1. Character Set @@ -800,7 +805,9 @@ class IO { virtual void endFlowMapping() = 0; virtual void beginEnumScalar() = 0; - virtual bool matchEnumScalar(const char*, bool) = 0; + virtual bool matchEnumScalar(const char*, + bool, + QuotingType=QuotingType::None) = 0; virtual bool matchEnumFallback() = 0; virtual void endEnumScalar() = 0; @@ -818,16 +825,24 @@ class IO { virtual void setAllowUnknownKeys(bool Allow); template - void enumCase(T &Val, const char* Str, const T ConstVal) { - if ( matchEnumScalar(Str, outputting() && Val == ConstVal) ) { + void enumCase(T &Val, + const char* Str, + const T ConstVal, + QuotingType MustQuote=QuotingType::None) { + if ( matchEnumScalar(Str, outputting() && Val == ConstVal, MustQuote) ) { Val = ConstVal; } } // allow anonymous enum values to be used with LLVM_YAML_STRONG_TYPEDEF template - void enumCase(T &Val, const char* Str, const uint32_t ConstVal) { - if ( matchEnumScalar(Str, outputting() && Val == static_cast(ConstVal)) ) { + void enumCase(T &Val, + const char* Str, + const uint32_t ConstVal, + QuotingType MustQuote=QuotingType::None) { + if ( matchEnumScalar(Str, + outputting() && Val == static_cast(ConstVal), + MustQuote) ) { Val = ConstVal; } } @@ -1156,16 +1171,42 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) { char missing_yaml_trait_for_type[sizeof(MissingTrait)]; } +template +class DefaultInserter { +private: + IO &io; + T &Seq; + +public: + DefaultInserter(IO &io, T &Seq) : io(io), Seq(Seq) {} + + auto &preflightElement(unsigned i) { + return SequenceTraits::element(io, Seq, i); + } + + void postflightElement(unsigned i) {}; +}; + +template +auto getInserter(const T &) -> typename SequenceTraits::Inserter { + return std::declval::Inserter>(); +} + +template +DefaultInserter getInserter(...) { return DefaultInserter{}; } + template std::enable_if_t::value, void> yamlize(IO &io, T &Seq, bool, Context &Ctx) { + decltype(getInserter(std::declval())) I(io, Seq); if ( has_FlowTraits< SequenceTraits>::value ) { unsigned incnt = io.beginFlowSequence(); unsigned count = io.outputting() ? SequenceTraits::size(io, Seq) : incnt; for(unsigned i=0; i < count; ++i) { void *SaveInfo; if ( io.preflightFlowElement(i, SaveInfo) ) { - yamlize(io, SequenceTraits::element(io, Seq, i), true, Ctx); + yamlize(io, I.preflightElement(i), true, Ctx); + I.postflightElement(i); io.postflightFlowElement(SaveInfo); } } @@ -1177,7 +1218,8 @@ yamlize(IO &io, T &Seq, bool, Context &Ctx) { for(unsigned i=0; i < count; ++i) { void *SaveInfo; if ( io.preflightElement(i, SaveInfo) ) { - yamlize(io, SequenceTraits::element(io, Seq, i), true, Ctx); + yamlize(io, I.preflightElement(i), true, Ctx); + I.postflightElement(i); io.postflightElement(SaveInfo); } } @@ -1452,7 +1494,9 @@ class Input : public IO { void postflightFlowElement(void *) override; void endFlowSequence() override; void beginEnumScalar() override; - bool matchEnumScalar(const char*, bool) override; + bool matchEnumScalar(const char*, + bool, + QuotingType=QuotingType::None) override; bool matchEnumFallback() override; void endEnumScalar() override; bool beginBitSetScalar(bool &) override; @@ -1608,7 +1652,9 @@ class Output : public IO { void postflightFlowElement(void *) override; void endFlowSequence() override; void beginEnumScalar() override; - bool matchEnumScalar(const char*, bool) override; + bool matchEnumScalar(const char*, + bool, + QuotingType=QuotingType::None) override; bool matchEnumFallback() override; void endEnumScalar() override; bool beginBitSetScalar(bool &) override; diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h index a876385581e7..d49c9039b0e8 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h @@ -30,6 +30,8 @@ #define DEBUG_TYPE "instcombine" #include "llvm/Transforms/Utils/InstructionWorklist.h" +extern llvm::cl::opt MaxArraySize; + namespace llvm { class AAResults; diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h index 26348da22021..a50b9bc51161 100644 --- a/llvm/include/llvm/Transforms/Scalar/SROA.h +++ b/llvm/include/llvm/Transforms/Scalar/SROA.h @@ -21,8 +21,11 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/CommandLine.h" #include +extern llvm::cl::opt SROANoArrays; + namespace llvm { class AllocaInst; diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 77050391746b..e1a14a7a5f8c 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -57,6 +57,21 @@ std::unique_ptr CloneModule(const Module &M, ValueToValueMapTy &VMap, function_ref ShouldCloneDefinition); +enum class CloneAction : uint8_t { + Omit, + MakeDeclaration, + Clone, +}; + +/// This version works similarly to the one above with `ShouldCloneDefinition`, +/// however it allows specifying that a GlobalValue is to be completely skipped +/// for cloning via `CloneAction::Omit`. +/// NOTE: passing `Omit` is dangerous, if a GV is omitted and it's used by e.g. +/// a function then it will lead to an assertion when cloning. +std::unique_ptr +CloneModule(const Module &M, ValueToValueMapTy &VMap, + function_ref Action); + /// This struct can be used to capture information about code /// being cloned, while it is being cloned. struct ClonedCodeInfo { diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index d63bee6fa321..e2af52956db8 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -19,7 +19,15 @@ namespace llvm { -template class DomTreeNodeBase; +template +using DTIdentityView = X; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; + using DomTreeNode = DomTreeNodeBase; class AssumptionCache; class StringRef; diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h index ff60811b6168..cd089574cc7d 100644 --- a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h +++ b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h @@ -22,12 +22,14 @@ class WeakTrackingVH; template class SmallVectorImpl; class CastInst; class DominatorTree; +class Instruction; class Loop; class LoopInfo; class PHINode; class ScalarEvolution; class SCEVExpander; class TargetTransformInfo; +class Value; /// Interface for visiting interesting IV users that are recognized but not /// simplified by this utility. @@ -58,6 +60,10 @@ bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, DominatorTree *DT, LoopInfo *LI, const TargetTransformInfo *TTI, SmallVectorImpl &Dead); +/// We make it public so it can be used in our LLVM out-of-tree Passes. +Instruction *getInsertPointForUses(Instruction *User, Value *Def, + DominatorTree *DT, LoopInfo *LI); + /// Collect information about induction variables that are used by sign/zero /// extend operations. This information is recorded by CollectExtend and provides /// the input to WidenIV. diff --git a/llvm/lib/Analysis/CallGraphSCCPass.cpp b/llvm/lib/Analysis/CallGraphSCCPass.cpp index d66f1e261780..564056c4ad4a 100644 --- a/llvm/lib/Analysis/CallGraphSCCPass.cpp +++ b/llvm/lib/Analysis/CallGraphSCCPass.cpp @@ -31,6 +31,7 @@ #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Progress.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" #include @@ -81,6 +82,8 @@ class CGPassManager : public ModulePass, public PMDataManager { StringRef getPassName() const override { return "CallGraph Pass Manager"; } + bool isSingleTask() const override { return true; } + PMDataManager *getAsPMDataManager() override { return this; } Pass *getAsPass() override { return this; } @@ -171,14 +174,18 @@ bool CGPassManager::RunPassOnSCC(Pass *P, CallGraphSCC &CurSCC, FPPassManager *FPP = (FPPassManager*)P; // Run pass P on all functions in the current SCC. + Task T(CurSCC.size(), "Run function pass manager on SCC"); for (CallGraphNode *CGN : CurSCC) { if (Function *F = CGN->getFunction()) { + T.advance(F->getName(), true); dumpPassInfo(P, EXECUTION_MSG, ON_FUNCTION_MSG, F->getName()); { TimeRegion PassTimer(getPassTimer(FPP)); Changed |= FPP->runOnFunction(*F); } F->getContext().yield(); + } else { + T.advance(); } } @@ -436,10 +443,13 @@ bool CGPassManager::RunAllPassesOnSCC(CallGraphSCC &CurSCC, CallGraph &CG, // the callgraph when we need to run a CGSCCPass again. bool CallGraphUpToDate = true; + Task T(getNumContainedPasses(), "Run passes on SCC"); + // Run all passes on current SCC. for (unsigned PassNo = 0, e = getNumContainedPasses(); PassNo != e; ++PassNo) { Pass *P = getContainedPass(PassNo); + T.advance(P->getPassName(), P->isSingleTask()); // If we're in -debug-pass=Executions mode, construct the SCC node list, // otherwise avoid constructing this string as it is expensive. @@ -502,8 +512,10 @@ bool CGPassManager::runOnModule(Module &M) { bool Changed = doInitialization(CG); // Walk the callgraph in bottom-up SCC order. - scc_iterator CGI = scc_begin(&CG); + auto T = make_task_on_set(make_range(scc_begin(&CG), scc_end(&CG)), + "LLVM SCC Passes"); + scc_iterator CGI = scc_begin(&CG); CallGraphSCC CurSCC(CG, &CGI); while (!CGI.isAtEnd()) { // Copy the current SCC and increment past it so that the pass can hack @@ -511,6 +523,7 @@ bool CGPassManager::runOnModule(Module &M) { const std::vector &NodeVec = *CGI; CurSCC.initialize(NodeVec); ++CGI; + T.advance(NodeVec, "", true); // At the top level, we run all the passes in this pass manager on the // functions in this SCC. However, we support iterative compilation in the @@ -524,9 +537,11 @@ bool CGPassManager::runOnModule(Module &M) { // This only happens in the case of a devirtualized call, so we only burn // compile time in the case that we're making progress. We also have a hard // iteration count limit in case there is crazy code. + Task SCCT({}, "Fixed-point SCC visit"); unsigned Iteration = 0; bool DevirtualizedCall = false; do { + SCCT.advance("", true); LLVM_DEBUG(if (Iteration) dbgs() << " SCCPASSMGR: Re-visiting SCC, iteration #" << Iteration << '\n'); diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index c83eb96bbc69..ff535811b566 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -6515,12 +6515,21 @@ Value *llvm::simplifyCall(CallBase *Call, const SimplifyQuery &Q) { if (Call->isMustTailCall()) return nullptr; - // call undef -> poison - // call null -> poison Value *Callee = Call->getCalledOperand(); - if (isa(Callee) || isa(Callee)) + + // call undef -> poison + if (isa(Callee)) return PoisonValue::get(Call->getType()); + // call null -> poison, but only if Call is in a function where null pointer + // is undefined + if (isa(Callee)) { + if (!Call->getParent() || !Call->getFunction() || + !NullPointerIsDefined(Call->getFunction())) { + return PoisonValue::get(Call->getType()); + } + } + if (Value *V = tryConstantFoldCall(Call, Q)) return V; diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp index 0e9fab667e6e..944b4a2938da 100644 --- a/llvm/lib/Analysis/LazyValueInfo.cpp +++ b/llvm/lib/Analysis/LazyValueInfo.cpp @@ -990,10 +990,11 @@ LazyValueInfoImpl::solveBlockValueOverflowIntrinsic(WithOverflowInst *WO, std::optional LazyValueInfoImpl::solveBlockValueIntrinsic(IntrinsicInst *II, BasicBlock *BB) { + ValueLatticeElement MetadataVal = getFromRangeMetadata(II); if (!ConstantRange::isIntrinsicSupported(II->getIntrinsicID())) { LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName() << "' - unknown intrinsic.\n"); - return getFromRangeMetadata(II); + return MetadataVal; } SmallVector OpRanges; @@ -1004,8 +1005,9 @@ LazyValueInfoImpl::solveBlockValueIntrinsic(IntrinsicInst *II, BasicBlock *BB) { OpRanges.push_back(*Range); } - return ValueLatticeElement::getRange( - ConstantRange::intrinsic(II->getIntrinsicID(), OpRanges)); + return intersect(ValueLatticeElement::getRange(ConstantRange::intrinsic( + II->getIntrinsicID(), OpRanges)), + MetadataVal); } std::optional diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp index 9ad60f774e9f..9512ef50650d 100644 --- a/llvm/lib/Analysis/MemorySSAUpdater.cpp +++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp @@ -877,7 +877,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, // Check number of predecessors, we only care if there's more than one. unsigned Count = 0; BasicBlock *Pred = nullptr; - for (auto *Pi : GD->template getChildren(BB)) { + for (auto *Pi : GD->template getChildren(BB)) { Pred = Pi; Count++; if (Count == 2) @@ -971,7 +971,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, auto *BB = BBPredPair.first; const auto &AddedBlockSet = BBPredPair.second.Added; auto &PrevBlockSet = BBPredPair.second.Prev; - for (auto *Pi : GD->template getChildren(BB)) { + for (auto *Pi : GD->template getChildren(BB)) { if (!AddedBlockSet.count(Pi)) PrevBlockSet.insert(Pi); EdgeCountMap[{Pi, BB}]++; @@ -1122,7 +1122,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, for (unsigned I = 0, E = IDFPhi->getNumIncomingValues(); I < E; ++I) IDFPhi->setIncomingValue(I, GetLastDef(IDFPhi->getIncomingBlock(I))); } else { - for (auto *Pi : GD->template getChildren(BBIDF)) + for (auto *Pi : GD->template getChildren(BBIDF)) IDFPhi->addIncoming(GetLastDef(Pi), Pi); } } diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index 0a67c4b6beb6..31bf54b3d845 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -1684,6 +1684,9 @@ TypeIndex CodeViewDebug::lowerTypeAlias(const DIDerivedType *Ty) { addToUDTs(Ty); + AliasRecord AR(UnderlyingTypeIndex, TypeName); + auto AliasIndex = TypeTable.writeLeafType(AR); + if (UnderlyingTypeIndex == TypeIndex(SimpleTypeKind::Int32Long) && TypeName == "HRESULT") return TypeIndex(SimpleTypeKind::HResult); @@ -1691,7 +1694,7 @@ TypeIndex CodeViewDebug::lowerTypeAlias(const DIDerivedType *Ty) { TypeName == "wchar_t") return TypeIndex(SimpleTypeKind::WideCharacter); - return UnderlyingTypeIndex; + return AliasIndex; } TypeIndex CodeViewDebug::lowerTypeArray(const DICompositeType *Ty) { @@ -2703,14 +2706,6 @@ TypeIndex CodeViewDebug::getCompleteTypeIndex(const DIType *Ty) { if (!Ty) return TypeIndex::Void(); - // Look through typedefs when getting the complete type index. Call - // getTypeIndex on the typdef to ensure that any UDTs are accumulated and are - // emitted only once. - if (Ty->getTag() == dwarf::DW_TAG_typedef) - (void)getTypeIndex(Ty); - while (Ty->getTag() == dwarf::DW_TAG_typedef) - Ty = cast(Ty)->getBaseType(); - // If this is a non-record type, the complete type index is the same as the // normal type index. Just call getTypeIndex. switch (Ty->getTag()) { @@ -3298,6 +3293,11 @@ void CodeViewDebug::emitDebugInfoForRetainedTypes() { if (DIType *RT = dyn_cast(Ty)) { getTypeIndex(RT); // FIXME: Add to global/local DTU list. + } else if (auto *SP = dyn_cast(Ty)) { + // Emit LF_FUNC_ID for declaration-only subprograms so that + // function prototypes from headers appear in the PDB. + if (!SP->isDefinition()) + getFuncIdForSubprogram(SP); } } } diff --git a/llvm/lib/CodeGen/MachineDominators.cpp b/llvm/lib/CodeGen/MachineDominators.cpp index 0632cde9c6f4..62649cfdbcf0 100644 --- a/llvm/lib/CodeGen/MachineDominators.cpp +++ b/llvm/lib/CodeGen/MachineDominators.cpp @@ -35,8 +35,8 @@ static cl::opt VerifyMachineDomInfoX( cl::desc("Verify machine dominator info (time consuming)")); namespace llvm { -template class DomTreeNodeBase; -template class DominatorTreeBase; // DomTreeBase +template class DomTreeNodeOnView; +template class DominatorTreeOnView; // DomTreeBase } char MachineDominatorTree::ID = 0; diff --git a/llvm/lib/CodeGen/MachinePostDominators.cpp b/llvm/lib/CodeGen/MachinePostDominators.cpp index fb96d0efa4d4..c13fdbbc1815 100644 --- a/llvm/lib/CodeGen/MachinePostDominators.cpp +++ b/llvm/lib/CodeGen/MachinePostDominators.cpp @@ -17,7 +17,7 @@ using namespace llvm; namespace llvm { -template class DominatorTreeBase; // PostDomTreeBase +template class DominatorTreeOnView; // PostDomTreeBase extern bool VerifyMachineDomInfo; } // namespace llvm diff --git a/llvm/lib/DebugInfo/CodeView/RecordName.cpp b/llvm/lib/DebugInfo/CodeView/RecordName.cpp index 5fbbc4a5d497..bc5aa4670e01 100644 --- a/llvm/lib/DebugInfo/CodeView/RecordName.cpp +++ b/llvm/lib/DebugInfo/CodeView/RecordName.cpp @@ -251,6 +251,11 @@ Error TypeNameComputer::visitKnownRecord(CVType &CVR, return Error::success(); } +Error TypeNameComputer::visitKnownRecord(CVType &CVR, AliasRecord &Alias) { + Name = Alias.Name; + return Error::success(); +} + std::string llvm::codeview::computeTypeName(TypeCollection &Types, TypeIndex Index) { TypeNameComputer Computer(Types); diff --git a/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp b/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp index df7e42df1afc..f2f7bc6ceb44 100644 --- a/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp +++ b/llvm/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp @@ -567,3 +567,9 @@ Error TypeDumpVisitor::visitKnownRecord(CVType &CVR, W->printHex("Signature", EndPrecomp.getSignature()); return Error::success(); } + +Error TypeDumpVisitor::visitKnownRecord(CVType &CVR, AliasRecord &Alias) { + printTypeIndex("UnderlyingType", Alias.UnderlyingType); + W->printString("Name", Alias.Name); + return Error::success(); +} diff --git a/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp b/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp index 682747a2b81f..9396f2ac341a 100644 --- a/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp +++ b/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp @@ -308,6 +308,9 @@ static void discoverTypeIndices(ArrayRef Content, TypeLeafKind Kind, case TypeLeafKind::LF_MODIFIER: Refs.push_back({TiRefKind::TypeRef, 0, 1}); break; + case TypeLeafKind::LF_ALIAS: + Refs.push_back({TiRefKind::TypeRef, 0, 1}); + break; case TypeLeafKind::LF_PROCEDURE: Refs.push_back({TiRefKind::TypeRef, 0, 1}); Refs.push_back({TiRefKind::TypeRef, 8, 1}); diff --git a/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp b/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp index 0bc65f8d0359..eaebd0865f62 100644 --- a/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp +++ b/llvm/lib/DebugInfo/CodeView/TypeRecordMapping.cpp @@ -752,3 +752,9 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, error(IO.mapInteger(EndPrecomp.Signature, "Signature")); return Error::success(); } + +Error TypeRecordMapping::visitKnownRecord(CVType &CVR, AliasRecord &Alias) { + error(IO.mapInteger(Alias.UnderlyingType, "UnderlyingType")); + error(IO.mapStringZ(Alias.Name, "Name")); + return Error::success(); +} diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 6108ce09c289..ed7fe2cf07dd 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -88,6 +88,14 @@ using namespace llvm; +static cl::opt +EmitHexConstantLiteralsFrom("emit-hex-constant-literals-from", + cl::desc("Emit constant literals greater than or" + " equal to a certain unsigned value in" + " hexadecimal in LLVM IR."), + cl::init(std::numeric_limits::max())); + + // Make virtual table appear in this compilation unit. AssemblyAnnotationWriter::~AssemblyAnnotationWriter() = default; @@ -1353,7 +1361,16 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, Out << (CI->getZExtValue() ? "true" : "false"); return; } - Out << CI->getValue(); + + if (EmitHexConstantLiteralsFrom != std::numeric_limits::max() + && CI->getValue().uge(EmitHexConstantLiteralsFrom)) { + std::string Value = llvm::toString(CI->getValue(), 16, false, false); + llvm::transform(Value, Value.begin(), tolower); + Out << "u0x" << Value; + } else { + Out << CI->getValue(); + } + return; } diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index 0dbccaa1a66a..ab94053df45a 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -945,6 +945,7 @@ bool ConstantRange::isIntrinsicSupported(Intrinsic::ID IntrinsicID) { case Intrinsic::smin: case Intrinsic::smax: case Intrinsic::abs: + case Intrinsic::ctlz: return true; default: return false; @@ -976,6 +977,12 @@ ConstantRange ConstantRange::intrinsic(Intrinsic::ID IntrinsicID, assert(IntMinIsPoison->getBitWidth() == 1 && "Must be boolean"); return Ops[0].abs(IntMinIsPoison->getBoolValue()); } + case Intrinsic::ctlz: { + const APInt *ZeroIsPoison = Ops[1].getSingleElement(); + assert(ZeroIsPoison && "Must be known (immarg)"); + assert(ZeroIsPoison->getBitWidth() == 1 && "Must be boolean"); + return Ops[0].ctlz(ZeroIsPoison->getBoolValue()); + } default: assert(!isIntrinsicSupported(IntrinsicID) && "Shouldn't be supported"); llvm_unreachable("Unsupported intrinsic"); @@ -1667,6 +1674,45 @@ ConstantRange ConstantRange::abs(bool IntMinIsPoison) const { APIntOps::umax(-SMin, SMax) + 1); } +ConstantRange ConstantRange::ctlz(bool ZeroIsPoison) const { + if (isEmptySet()) + return getEmpty(); + + APInt Zero = APInt::getZero(getBitWidth()); + if (ZeroIsPoison && contains(Zero)) { + // ZeroIsPoison is set, and zero is contained. We discern three cases, in + // which a zero can appear: + // 1) Lower is zero, handling cases of kind [0, 1), [0, 2), etc. + // 2) Upper is zero, wrapped set, handling cases of kind [3, 0], etc. + // 3) Zero contained in a wrapped set, e.g., [3, 2), [3, 1), etc. + + if (getLower().isZero()) { + if ((getUpper() - 1).isZero()) { + // We have in input interval of kind [0, 1). In this case we cannot + // really help but return empty-set. + return getEmpty(); + } + + // Compute the resulting range by excluding zero from Lower. + return ConstantRange( + APInt(getBitWidth(), (getUpper() - 1).countLeadingZeros()), + APInt(getBitWidth(), (getLower() + 1).countLeadingZeros() + 1)); + } else if ((getUpper() - 1).isZero()) { + // Compute the resulting range by excluding zero from Upper. + return ConstantRange( + Zero, APInt(getBitWidth(), getLower().countLeadingZeros() + 1)); + } else { + return ConstantRange(Zero, APInt(getBitWidth(), getBitWidth())); + } + } + + // Zero is either safe or not in the range. The output range is composed by + // the result of countLeadingZero of the two extremes. + return getNonEmpty( + APInt(getBitWidth(), getUnsignedMax().countLeadingZeros()), + APInt(getBitWidth(), getUnsignedMin().countLeadingZeros() + 1)); +} + ConstantRange::OverflowResult ConstantRange::unsignedAddMayOverflow( const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 9198179674bd..136815c7fa78 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -211,6 +211,12 @@ void DebugInfoFinder::processInstruction(const Module &M, if (auto DbgLoc = I.getDebugLoc()) processLocation(M, DbgLoc.get()); + + MDNode *LoopMD = I.getMetadata(LLVMContext::MD_loop); + if (auto *Tuple = cast_or_null(LoopMD)) + for (const MDOperand &Operand : Tuple->operands()) + if (const DILocation *Location = dyn_cast(Operand.get())) + processLocation(M, Location); } void DebugInfoFinder::processLocation(const Module &M, const DILocation *Loc) { diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp index 7c620c3a9331..9330508583aa 100644 --- a/llvm/lib/IR/Dominators.cpp +++ b/llvm/lib/IR/Dominators.cpp @@ -70,43 +70,59 @@ bool BasicBlockEdge::isSingleEdge() const { // //===----------------------------------------------------------------------===// -template class llvm::DomTreeNodeBase; -template class llvm::DominatorTreeBase; // DomTreeBase -template class llvm::DominatorTreeBase; // PostDomTreeBase + +namespace llvm { + +template class View> +class DominatorTreeOnView; + +template +using DominatorTreeBase = DominatorTreeOnView; + +template class View> +class DomTreeNodeOnView; + +template +using DomTreeNodeBase = DomTreeNodeOnView; + +template class DomTreeNodeOnView; +template class DominatorTreeOnView; // DomTree +template class DominatorTreeOnView; // PostDomTree +} // end namespace template class llvm::cfg::Update; -template void llvm::DomTreeBuilder::Calculate( +template void llvm::DomTreeBuilder::Calculate( DomTreeBuilder::BBDomTree &DT); template void -llvm::DomTreeBuilder::CalculateWithUpdates( +llvm::DomTreeBuilder::CalculateWithUpdates( DomTreeBuilder::BBDomTree &DT, BBUpdates U); -template void llvm::DomTreeBuilder::Calculate( +template void llvm::DomTreeBuilder::Calculate( DomTreeBuilder::BBPostDomTree &DT); // No CalculateWithUpdates instantiation, unless a usecase arises. -template void llvm::DomTreeBuilder::InsertEdge( +template void llvm::DomTreeBuilder::InsertEdge( DomTreeBuilder::BBDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::InsertEdge( +template void llvm::DomTreeBuilder::InsertEdge( DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::DeleteEdge( +template void llvm::DomTreeBuilder::DeleteEdge( DomTreeBuilder::BBDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::DeleteEdge( +template void llvm::DomTreeBuilder::DeleteEdge( DomTreeBuilder::BBPostDomTree &DT, BasicBlock *From, BasicBlock *To); -template void llvm::DomTreeBuilder::ApplyUpdates( - DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBDomTreeGraphDiff &, - DomTreeBuilder::BBDomTreeGraphDiff *); -template void llvm::DomTreeBuilder::ApplyUpdates( - DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBPostDomTreeGraphDiff &, - DomTreeBuilder::BBPostDomTreeGraphDiff *); +template void +llvm::DomTreeBuilder::ApplyUpdates( + BBDomTree &DT, BBDomTreeGraphDiff &, BBDomTreeGraphDiff *); +template void +llvm::DomTreeBuilder::ApplyUpdates( + BBPostDomTree &DT, BBPostDomTreeGraphDiff &, BBPostDomTreeGraphDiff *); -template bool llvm::DomTreeBuilder::Verify( +template bool llvm::DomTreeBuilder::Verify( const DomTreeBuilder::BBDomTree &DT, DomTreeBuilder::BBDomTree::VerificationLevel VL); -template bool llvm::DomTreeBuilder::Verify( +template bool llvm::DomTreeBuilder::Verify( const DomTreeBuilder::BBPostDomTree &DT, DomTreeBuilder::BBPostDomTree::VerificationLevel VL); diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp index ef3465177647..f097161120fd 100644 --- a/llvm/lib/IR/LegacyPassManager.cpp +++ b/llvm/lib/IR/LegacyPassManager.cpp @@ -12,6 +12,7 @@ #include "llvm/IR/LegacyPassManager.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LLVMContext.h" @@ -24,6 +25,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Progress.h" #include "llvm/Support/TimeProfiler.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" @@ -358,7 +360,9 @@ bool FunctionPassManagerImpl::run(Function &F) { bool Changed = false; initializeAllAnalysisInfo(); + Task T(getNumContainedManagers(), "Run function pass managers"); for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { + T.advance(Twine("FPM index: ") + Twine(Index)); Changed |= getContainedManager(Index)->runOnFunction(F); F.getContext().yield(); } @@ -423,6 +427,8 @@ class MPPassManager : public Pass, public PMDataManager { StringRef getPassName() const override { return "Module Pass Manager"; } + bool isSingleTask() const override { return true; } + PMDataManager *getAsPMDataManager() override { return this; } Pass *getAsPass() override { return this; } @@ -531,7 +537,9 @@ bool PassManagerImpl::run(Module &M) { Changed |= ImPass->doInitialization(M); initializeAllAnalysisInfo(); + Task T(getNumContainedManagers(), "Run module pass managers"); for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { + T.advance(Twine("MPM index: ") + Twine(Index)); Changed |= getContainedManager(Index)->runOnModule(M); M.getContext().yield(); } @@ -1410,8 +1418,10 @@ bool FPPassManager::runOnFunction(Function &F) { llvm::TimeTraceScope FunctionScope("OptFunction", F.getName()); + Task T(getNumContainedPasses(), "Run function passes"); for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { FunctionPass *FP = getContainedPass(Index); + T.advance(FP->getPassName(), FP->isSingleTask()); bool LocalChanged = false; llvm::TimeTraceScope PassScope("RunPass", FP->getPassName()); @@ -1472,8 +1482,11 @@ bool FPPassManager::runOnFunction(Function &F) { bool FPPassManager::runOnModule(Module &M) { bool Changed = false; - for (Function &F : M) + auto T = make_task_on_set(make_address_range(M), "Run function pass manager"); + for (Function &F : M) { + T.advance(&F, F.getName()); Changed |= runOnFunction(F); + } return Changed; } @@ -1525,8 +1538,10 @@ MPPassManager::runOnModule(Module &M) { if (EmitICRemark) InstrCount = initSizeRemarkInfo(M, FunctionToInstrCount); + Task T(getNumContainedPasses(), "Run module passes"); for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { ModulePass *MP = getContainedPass(Index); + T.advance(MP->getPassName(), MP->isSingleTask()); bool LocalChanged = false; dumpPassInfo(MP, EXECUTION_MSG, ON_MODULE_MSG, M.getModuleIdentifier()); diff --git a/llvm/lib/IR/Pass.cpp b/llvm/lib/IR/Pass.cpp index 716d9d546f4f..018131dbe381 100644 --- a/llvm/lib/IR/Pass.cpp +++ b/llvm/lib/IR/Pass.cpp @@ -86,6 +86,10 @@ StringRef Pass::getPassName() const { return "Unnamed pass: implement Pass::getPassName()"; } +bool Pass::isSingleTask() const { + return false; +} + void Pass::preparePassManager(PMStack &) { // By default, don't do anything. } diff --git a/llvm/lib/IRReader/IRReader.cpp b/llvm/lib/IRReader/IRReader.cpp index 7885c36a7987..ed9569383d5d 100644 --- a/llvm/lib/IRReader/IRReader.cpp +++ b/llvm/lib/IRReader/IRReader.cpp @@ -12,6 +12,7 @@ #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/Timer.h" @@ -66,9 +67,10 @@ std::unique_ptr llvm::getLazyIRFileModule(StringRef Filename, ShouldLazyLoadMetadata); } -std::unique_ptr llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, - LLVMContext &Context, - ParserCallbacks Callbacks) { +static std::unique_ptr parseIRImpl(MemoryBufferRef Buffer, + SMDiagnostic &Err, + LLVMContext &Context, + ParserCallbacks Callbacks) { NamedRegionTimer T(TimeIRParsingName, TimeIRParsingDescription, TimeIRParsingGroupName, TimeIRParsingGroupDescription, TimePassesIsEnabled); @@ -91,6 +93,36 @@ std::unique_ptr llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, [](StringRef, StringRef) { return std::nullopt; })); } +std::unique_ptr llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err, + LLVMContext &Context, + ParserCallbacks Callbacks) { + llvm::ArrayRef BufferRef( + reinterpret_cast(Buffer.getBufferStart()), + reinterpret_cast(Buffer.getBufferEnd())); + if (compression::zstd::isAvailable() && + compression::zstd::isZstd(BufferRef)) { + SmallVector DecompressedBuffer; + Error Error = compression::zstd::decompress(BufferRef, DecompressedBuffer); + if (Error) { + handleAllErrors(std::move(Error), [&](ErrorInfoBase &EIB) { + Err = SMDiagnostic(Buffer.getBufferIdentifier(), SourceMgr::DK_Error, + EIB.message()); + }); + return nullptr; + } + + std::string Identifier = + Buffer.getBufferIdentifier().str() + "-decompressed"; + MemoryBufferRef DecompressedBufferRef( + {reinterpret_cast(DecompressedBuffer.data()), + DecompressedBuffer.size()}, + Identifier); + return parseIRImpl(DecompressedBufferRef, Err, Context, Callbacks); + } else { + return parseIRImpl(Buffer, Err, Context, Callbacks); + } +} + std::unique_ptr llvm::parseIRFile(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, ParserCallbacks Callbacks) { diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp index 517e2dc8ebe0..462e824f4017 100644 --- a/llvm/lib/Linker/IRMover.cpp +++ b/llvm/lib/Linker/IRMover.cpp @@ -253,13 +253,6 @@ Type *TypeMapTy::get(Type *Ty, SmallPtrSet &Visited) { bool IsUniqued = !isa(Ty) || cast(Ty)->isLiteral(); if (!IsUniqued) { -#ifndef NDEBUG - for (auto &Pair : MappedTypes) { - assert(!(Pair.first != Ty && Pair.second == Ty) && - "mapping to a source type"); - } -#endif - if (!Visited.insert(cast(Ty)).second) { StructType *DTy = StructType::create(Ty->getContext()); return *Entry = DTy; diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp index 9c0b85cf7416..35d8a32fb7c5 100644 --- a/llvm/lib/Object/MachOObjectFile.cpp +++ b/llvm/lib/Object/MachOObjectFile.cpp @@ -3518,8 +3518,9 @@ void MachORebaseEntry::moveNext() { DEBUG_WITH_TYPE("mach-o-rebase", dbgs() << "REBASE_OPCODE_DONE\n"); break; case MachO::REBASE_OPCODE_SET_TYPE_IMM: - RebaseType = ImmValue; - if (RebaseType > MachO::REBASE_TYPE_TEXT_PCREL32) { + RebaseType = static_cast(ImmValue); + if (RebaseType <= MachO::REBASE_TYPE_INVALID + || RebaseType < MachO::REBASE_TYPE_POINTER) { *E = malformedError("for REBASE_OPCODE_SET_TYPE_IMM bad bind type: " + Twine((int)RebaseType) + " for opcode at: 0x" + Twine::utohexstr(OpcodeStart - Opcodes.begin())); @@ -3748,8 +3749,12 @@ int32_t MachORebaseEntry::segmentIndex() const { return SegmentIndex; } uint64_t MachORebaseEntry::segmentOffset() const { return SegmentOffset; } +MachO::RebaseType MachORebaseEntry::type() const { return RebaseType; } + StringRef MachORebaseEntry::typeName() const { switch (RebaseType) { + case MachO::REBASE_TYPE_INVALID: + return "invalid"; case MachO::REBASE_TYPE_POINTER: return "pointer"; case MachO::REBASE_TYPE_TEXT_ABSOLUTE32: @@ -3978,8 +3983,9 @@ void MachOBindEntry::moveNext() { } break; case MachO::BIND_OPCODE_SET_TYPE_IMM: - BindType = ImmValue; - if (ImmValue > MachO::BIND_TYPE_TEXT_PCREL32) { + BindType = static_cast(ImmValue); + if (ImmValue <= MachO::BIND_TYPE_INVALID + || ImmValue > MachO::BIND_TYPE_TEXT_PCREL32) { *E = malformedError("for BIND_OPCODE_SET_TYPE_IMM bad bind type: " + Twine((int)ImmValue) + " for opcode at: 0x" + Twine::utohexstr(OpcodeStart - Opcodes.begin())); @@ -4292,8 +4298,12 @@ int32_t MachOBindEntry::segmentIndex() const { return SegmentIndex; } uint64_t MachOBindEntry::segmentOffset() const { return SegmentOffset; } +MachO::BindType MachOBindEntry::type() const { return BindType; } + StringRef MachOBindEntry::typeName() const { switch (BindType) { + case MachO::BIND_TYPE_INVALID: + return "invalid"; case MachO::BIND_TYPE_POINTER: return "pointer"; case MachO::BIND_TYPE_TEXT_ABSOLUTE32: diff --git a/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp b/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp index e4e2b2a6d21a..a5c6fc2a7b9b 100644 --- a/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp +++ b/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp @@ -613,6 +613,11 @@ template <> void LeafRecordImpl::map(IO &IO) { IO.mapRequired("Signature", Record.Signature); } +template <> void LeafRecordImpl::map(IO &IO) { + IO.mapRequired("UnderlyingType", Record.UnderlyingType); + IO.mapRequired("Name", Record.Name); +} + template <> void MemberRecordImpl::map(IO &IO) { MappingTraits::mapping(IO, Record); } diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 4cbc3b79f3bb..57b7debfc078 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -154,6 +154,7 @@ add_llvm_component_library(LLVMSupport DataExtractor.cpp Debug.cpp DebugCounter.cpp + DebugInfoPreservation.cpp DeltaAlgorithm.cpp DivisionByConstantInfo.cpp DAGDeltaAlgorithm.cpp @@ -198,6 +199,7 @@ add_llvm_component_library(LLVMSupport Parallel.cpp PluginLoader.cpp PrettyStackTrace.cpp + Progress.cpp RandomNumberGenerator.cpp Regex.cpp RISCVAttributes.cpp diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp index 8e57ba798f52..9f148dbbf627 100644 --- a/llvm/lib/Support/Compression.cpp +++ b/llvm/lib/Support/Compression.cpp @@ -159,6 +159,11 @@ Error zlib::decompress(ArrayRef Input, } #endif +bool zstd::isZstd(ArrayRef Input) { + static uint8_t Magic[] = {0x28, 0xb5, 0x2f, 0xfd}; + return ::memcmp(Input.data(), Magic, 4) == 0; +} + #if LLVM_ENABLE_ZSTD bool zstd::isAvailable() { return true; } @@ -202,6 +207,78 @@ Error zstd::decompress(ArrayRef Input, return E; } +static void zstdContextFree(ZSTD_DCtx *Ctx) { + size_t RC = ::ZSTD_freeDCtx(Ctx); + assert(::ZSTD_isError(RC) == 0); +} + +static Expected +zstdDecompressRead(::ZSTD_DCtx *Ctx, ::ZSTD_inBuffer &ZSTDInput, + SmallVector &Buffer, + SmallVectorImpl &Decompressed) { + ::ZSTD_outBuffer ZSTDOutput = { + .dst = Buffer.data(), .size = Buffer.size(), .pos = 0}; + size_t Res = ::ZSTD_decompressStream(Ctx, &ZSTDOutput, &ZSTDInput); + if (::ZSTD_isError(Res)) { + return make_error(::ZSTD_getErrorName(Res), + inconvertibleErrorCode()); + } + + // Check if the buffer was filled, this might not happen in all iterations + // because the data read might still reside in the internal state of the + // decompressor + if (ZSTDOutput.pos > 0) { + Decompressed.append(Buffer.begin(), Buffer.begin() + ZSTDOutput.pos); + return true; + } + return false; +} + +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output) { + auto &Decompressed = Output; + std::unique_ptr<::ZSTD_DCtx, void (*)(::ZSTD_DCtx *)> Ctx(::ZSTD_createDCtx(), + zstdContextFree); + size_t Res = ::ZSTD_initDStream(&*Ctx); + if (::ZSTD_isError(Res)) + return make_error(::ZSTD_getErrorName(Res), + inconvertibleErrorCode()); + + // Create a buffer, this will be filled up repeatedly by + // ZSTD_decompressStream and appended to Output + SmallVector Buffer; + Buffer.resize_for_overwrite(4096); + + ::ZSTD_inBuffer ZSTDInput = { + .src = Input.data(), .size = Input.size(), .pos = 0}; + + // First loop, read the entirety of `InputBuffer`. Stop when + // ZSTD_decompressedStream reports pos to be equal to size. This signals that + // all the data has been read and is at least in the internal state of the + // decompressor. + while (ZSTDInput.pos < ZSTDInput.size) { + Expected Err = + zstdDecompressRead(&*Ctx, ZSTDInput, Buffer, Decompressed); + if (!Err) + return Err.takeError(); + } + + // Second loop, in this one there isn't any data to read, but there still + // might be data to be outputted from the decompressor internal state + ZSTDInput = {.src = nullptr, .size = 0, .pos = 0}; + while (true) { + Expected MaybeRes = + zstdDecompressRead(&*Ctx, ZSTDInput, Buffer, Decompressed); + if (!MaybeRes) + return MaybeRes.takeError(); + // If this if is taken it means that the decompress function did not write + // anything, which means that there isn't more data to decompress + if (!MaybeRes.get()) + break; + } + return Error::success(); +} + #else bool zstd::isAvailable() { return false; } void zstd::compress(ArrayRef Input, @@ -217,4 +294,8 @@ Error zstd::decompress(ArrayRef Input, size_t UncompressedSize) { llvm_unreachable("zstd::decompress is unavailable"); } +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output) { + llvm_unreachable("zstd::decompress is unavailable"); +} #endif diff --git a/llvm/lib/Support/DebugInfoPreservation.cpp b/llvm/lib/Support/DebugInfoPreservation.cpp new file mode 100644 index 000000000000..ff667ea3c029 --- /dev/null +++ b/llvm/lib/Support/DebugInfoPreservation.cpp @@ -0,0 +1,24 @@ +//===-- CommandLine.cpp - Command line parser implementation --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an extra command line option that changes debug +// information preservation logic accross multiple transformation utilities. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/DebugInfoPreservation.h" + +static constexpr llvm::StringRef Description = "Enable stricter debug " + "information preservation. In " + "particular avoid dropping it " + "even if it means conventional " + "debugger (gdb, lldb, etc) " + "experience degrades."; +llvm::cl::opt llvm::EnableStrictDebugInformationPreservationStyle( + "enable-strict-debug-information-preservation-style", + llvm::cl::desc(Description)); diff --git a/llvm/lib/Support/Error.cpp b/llvm/lib/Support/Error.cpp index fbe86f2b59e1..05e7144c06b4 100644 --- a/llvm/lib/Support/Error.cpp +++ b/llvm/lib/Support/Error.cpp @@ -177,3 +177,7 @@ LLVMErrorTypeId LLVMGetStringErrorTypeId() { LLVMErrorRef LLVMCreateStringError(const char *ErrMsg) { return wrap(make_error(ErrMsg, inconvertibleErrorCode())); } + +ExitOnError::ExitOnError(std::string Banner, int DefaultErrorExitCode) + : Banner(std::move(Banner)), + GetExitCode([=](const Error &) { return DefaultErrorExitCode; }) {} diff --git a/llvm/lib/Support/ErrorHandling.cpp b/llvm/lib/Support/ErrorHandling.cpp index b8b3b7424ac6..1959966858c9 100644 --- a/llvm/lib/Support/ErrorHandling.cpp +++ b/llvm/lib/Support/ErrorHandling.cpp @@ -19,11 +19,13 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Error.h" +#include "llvm/Support/Memory.h" #include "llvm/Support/Process.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Threading.h" #include "llvm/Support/WindowsError.h" #include "llvm/Support/raw_ostream.h" +#include #include #include #include @@ -39,12 +41,38 @@ using namespace llvm; +class MemoryReserver { +private: + std::error_code EC; + sys::MemoryBlock Ptr; + +public: + MemoryReserver(size_t Pages): Ptr() { + using sys::Memory; + unsigned int PSize = sys::Process::getPageSizeEstimate(); + Ptr = Memory::allocateMappedMemory(PSize * Pages, nullptr, Memory::MF_READ | Memory::MF_WRITE, EC); + assert(!EC); + } + + ~MemoryReserver() { + release(); + } + + void release() { + EC = sys::Memory::releaseMappedMemory(Ptr); + assert(!EC); + } +}; + static fatal_error_handler_t ErrorHandler = nullptr; static void *ErrorHandlerUserData = nullptr; static fatal_error_handler_t BadAllocErrorHandler = nullptr; static void *BadAllocErrorHandlerUserData = nullptr; +static MemoryReserver OOMReservedMemory(1024); +static std::atomic_bool BadAllocCalled(false); + #if LLVM_ENABLE_THREADS == 1 // Mutexes to synchronize installing error handlers and calling error handlers. // Do not use ManagedStatic, or that may allocate memory while attempting to @@ -146,12 +174,14 @@ void llvm::remove_bad_alloc_error_handler() { void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) { fatal_error_handler_t Handler = nullptr; void *HandlerData = nullptr; + bool BadAllocCalledPrev = BadAllocCalled.exchange(true); { // Only acquire the mutex while reading the handler, so as not to invoke a // user-supplied callback under a lock. #if LLVM_ENABLE_THREADS == 1 std::lock_guard Lock(BadAllocErrorHandlerMutex); #endif + OOMReservedMemory.release(); Handler = BadAllocErrorHandler; HandlerData = BadAllocErrorHandlerUserData; } @@ -172,7 +202,10 @@ void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) { (void)!::write(2, OOMMessage, strlen(OOMMessage)); (void)!::write(2, Reason, strlen(Reason)); (void)!::write(2, Newline, strlen(Newline)); - abort(); + if (not BadAllocCalledPrev) { + sys::PrintStackTrace(llvm::errs()); + } + ::_Exit(235); #endif } diff --git a/llvm/lib/Support/Progress.cpp b/llvm/lib/Support/Progress.cpp new file mode 100644 index 000000000000..ea7f74b8e9bb --- /dev/null +++ b/llvm/lib/Support/Progress.cpp @@ -0,0 +1,116 @@ +//===-- Progress - Generic infrastructure to report progress ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the core logic for progress reporting. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/Progress.h" + +using namespace llvm; + +ManagedStatic llvm::ProgressReport; + +static thread_local TaskStack CurrentStack(*ProgressReport); + +Progress::Progress() : MainThreadID(get_threadid()) {} + +bool Progress::isMainThread() const { return MainThreadID == get_threadid(); } + +void Progress::handleNewTask(const Task *T) { + bool IsMainThread = isMainThread(); + for (auto &[AllThreads, Listener] : Registry) + if (IsMainThread or AllThreads) + Listener->handleNewTask(T); +} + +void Progress::handleTaskCompleted(const Task *T) { + bool IsMainThread = isMainThread(); + for (auto &[AllThreads, Listener] : Registry) + if (IsMainThread or AllThreads) + Listener->handleTaskCompleted(T); +} + +void Progress::handleTaskAdvancement(const Task *T, + StringRef PreviousStepName) { + bool IsMainThread = isMainThread(); + for (auto &[AllThreads, Listener] : Registry) + if (IsMainThread or AllThreads) + Listener->handleTaskAdvancement(T, PreviousStepName); +} + +Task::Task(std::optional TotalSteps, const Twine &TaskName) + : Stack(CurrentStack), TotalSteps(TotalSteps), TaskName(TaskName.str()) { + Stack.registerTask(this); +} + +void Task::complete() { + if (not Completed) { + Completed = true; + Stack.unregisterTask(this); + } +} + +Task::~Task() { complete(); } + +void Task::advance(const Twine &NewStepName, bool SingleSubtask) { + int64_t NewStepIndex = StepIndex + 1; + if (TotalSteps.has_value()) { + assert(NewStepIndex < *TotalSteps); + } + StepIndex = NewStepIndex; + std::string PreviousStepName = std::move(StepName); + StepName = NewStepName.str(); + SubtaskCreated = false; + CurrentStepHasSingleSubtask = SingleSubtask; + + Stack.advanceTask(this, PreviousStepName); +} + +void Task::setSubtaskCreated() { SubtaskCreated = true; } + +size_t Task::index() const { + auto &Stack = stack(); + auto Begin = Stack.Tasks.begin(); + auto End = Stack.Tasks.end(); + auto It = std::find(Begin, End, this); + assert(It != End); + return It - Begin; +} + +void TaskStack::registerTask(Task *T) { + if (isSuspended()) + return; + + if (Tasks.size() > 0) { + if (T->currentStepHasSingleSubtask()) { + assert(not Tasks.back()->subtaskCreated()); + } + Tasks.back()->setSubtaskCreated(); + } + + Tasks.push_back(T); + + Parent.handleNewTask(T); +} + +void TaskStack::unregisterTask(Task *T) { + if (isSuspended()) + return; + + assert(Tasks.back() == T); + Parent.handleTaskCompleted(T); + Tasks.pop_back(); +} + +void TaskStack::advanceTask(Task *T, StringRef PreviousStepName) { + if (isSuspended()) + return; + + Parent.handleTaskAdvancement(T, PreviousStepName); +} diff --git a/llvm/lib/Support/Unix/Signals.inc b/llvm/lib/Support/Unix/Signals.inc index 05a7335216f4..374022389594 100644 --- a/llvm/lib/Support/Unix/Signals.inc +++ b/llvm/lib/Support/Unix/Signals.inc @@ -259,8 +259,8 @@ static struct { // reported as a leak. We don't make any attempt to remove our alt signal // stack if we remove our signal handlers; that can't be done reliably if // someone else is also trying to do the same thing. -static stack_t OldAltStack; -LLVM_ATTRIBUTE_USED static void *NewAltStackPointer; +thread_local static stack_t OldAltStack; +LLVM_ATTRIBUTE_USED thread_local static void* NewAltStackPointer; static void CreateSigAltStack() { const size_t AltStackSize = MINSIGSTKSZ + 64 * 1024; @@ -276,10 +276,13 @@ static void CreateSigAltStack() { stack_t AltStack = {}; AltStack.ss_sp = static_cast(safe_malloc(AltStackSize)); - NewAltStackPointer = AltStack.ss_sp; // Save to avoid reporting a leak. AltStack.ss_size = AltStackSize; - if (sigaltstack(&AltStack, &OldAltStack) != 0) + if (sigaltstack(&AltStack, &OldAltStack) != 0) { free(AltStack.ss_sp); + } else { + free(NewAltStackPointer); + NewAltStackPointer = AltStack.ss_sp; + } } #else static void CreateSigAltStack() {} diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc index 819748db4ec2..81176f0a5822 100644 --- a/llvm/lib/Support/Unix/Threading.inc +++ b/llvm/lib/Support/Unix/Threading.inc @@ -127,8 +127,6 @@ uint64_t llvm::get_threadid() { #elif defined(__ANDROID__) return uint64_t(gettid()); #elif defined(__linux__) - return uint64_t(syscall(SYS_gettid)); -#else return uint64_t(pthread_self()); #endif } diff --git a/llvm/lib/Support/YAMLTraits.cpp b/llvm/lib/Support/YAMLTraits.cpp index 4eb0b3afd563..d585b5507df1 100644 --- a/llvm/lib/Support/YAMLTraits.cpp +++ b/llvm/lib/Support/YAMLTraits.cpp @@ -271,7 +271,7 @@ void Input::beginEnumScalar() { ScalarMatchFound = false; } -bool Input::matchEnumScalar(const char *Str, bool) { +bool Input::matchEnumScalar(const char *Str, bool, QuotingType MustQuote) { if (ScalarMatchFound) return false; if (ScalarHNode *SN = dyn_cast(CurrentNode)) { @@ -650,10 +650,11 @@ void Output::beginEnumScalar() { EnumerationMatchFound = false; } -bool Output::matchEnumScalar(const char *Str, bool Match) { +bool Output::matchEnumScalar(const char *Str, bool Match, QuotingType MustQuote) { if (Match && !EnumerationMatchFound) { newLineCheck(); - outputUpToEndOfLine(Str); + StringRef String(Str); + scalarString(String, MustQuote); EnumerationMatchFound = true; } return false; diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp index 6ff5459fe026..75bf7633e852 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp @@ -1751,12 +1751,11 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address, // If the label has already been resolved to an immediate offset (say, when // we're running the disassembler), just print the immediate. if (Op.isImm()) { - O << markup(""); return; } @@ -1766,7 +1765,7 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address, dyn_cast(MI->getOperand(OpNum).getExpr()); int64_t TargetAddress; if (BranchTarget && BranchTarget->evaluateAsAbsolute(TargetAddress)) { - O << formatHex((uint64_t)TargetAddress); + O << markup(""); } else { // Otherwise, just print the expression. MI->getOperand(OpNum).getExpr()->print(O, &MAI); @@ -1782,12 +1781,11 @@ void AArch64InstPrinter::printAdrpLabel(const MCInst *MI, uint64_t Address, // If the label has already been resolved to an immediate offset (say, when // we're running the disassembler), just print the immediate. if (Op.isImm()) { - const int64_t Offset = Op.getImm() * 4096; - O << markup(""); return; } diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td index f5415c5b5895..15536b7c47f0 100644 --- a/llvm/lib/Target/ARM/ARMInstrInfo.td +++ b/llvm/lib/Target/ARM/ARMInstrInfo.td @@ -556,6 +556,7 @@ def brtarget : Operand { let EncoderMethod = "getBranchTargetOpValue"; let OperandType = "OPERAND_PCREL"; let DecoderMethod = "DecodeT2BROperand"; + let PrintMethod = "printPCRelativeOperand"; } // Branches targeting ARM-mode must be divisible by 4 if they're a raw @@ -574,6 +575,7 @@ def arm_br_target : Operand { let ParserMatchClass = ARMBranchTarget; let EncoderMethod = "getARMBranchTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } // Call target for ARM. Handles conditional/unconditional @@ -582,6 +584,7 @@ def arm_bl_target : Operand { let ParserMatchClass = ARMBranchTarget; let EncoderMethod = "getARMBLTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } // Target for BLX *from* ARM mode. @@ -589,6 +592,7 @@ def arm_blx_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getARMBLXTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } // A list of registers separated by comma. Used by load/store multiple. diff --git a/llvm/lib/Target/ARM/ARMInstrThumb.td b/llvm/lib/Target/ARM/ARMInstrThumb.td index 8f7039a327b3..9e7108c396a7 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb.td @@ -128,6 +128,7 @@ let OperandType = "OPERAND_PCREL" in { def t_brtarget : Operand { let EncoderMethod = "getThumbBRTargetOpValue"; let DecoderMethod = "DecodeThumbBROperand"; + let PrintMethod = "printPCRelativeOperand"; } // ADR instruction labels. @@ -142,12 +143,14 @@ def thumb_br_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbBranchTargetOpValue"; let OperandType = "OPERAND_PCREL"; + let PrintMethod = "printPCRelativeOperand"; } def thumb_bl_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbBLTargetOpValue"; let DecoderMethod = "DecodeThumbBLTargetOperand"; + let PrintMethod = "printPCRelativeOperand"; } // Target for BLX *from* thumb mode. @@ -155,18 +158,21 @@ def thumb_blx_target : Operand { let ParserMatchClass = ARMBranchTarget; let EncoderMethod = "getThumbBLXTargetOpValue"; let DecoderMethod = "DecodeThumbBLXOffset"; + let PrintMethod = "printPCRelativeOperand"; } def thumb_bcc_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbBCCTargetOpValue"; let DecoderMethod = "DecodeThumbBCCTargetOperand"; + let PrintMethod = "printPCRelativeOperand"; } def thumb_cb_target : Operand { let ParserMatchClass = ThumbBranchTarget; let EncoderMethod = "getThumbCBTargetOpValue"; let DecoderMethod = "DecodeThumbCmpBROperand"; + let PrintMethod = "printPCRelativeOperand"; } } // OperandType = "OPERAND_PCREL" diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td index 521cb0695384..74ac17f1ed38 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb2.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -376,6 +376,7 @@ class BFLabelOp"); + let PrintMethod = "printPCRelativeOperand"; } def bflabel_u4 : BFLabelOp<"false", "false", "false", "4", "ARM::fixup_bf_branch">; def bflabel_s12 : BFLabelOp<"true", "false", "true", "12", "ARM::fixup_bfc_target">; @@ -407,6 +408,7 @@ def bfafter_target : Operand { let EncoderMethod = "getBFAfterTargetOpValue"; let OperandType = "OPERAND_PCREL"; let DecoderMethod = "DecodeBFAfterTargetOperand"; + let PrintMethod = "printPCRelativeOperand"; } // pred operand excluding AL diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp index 9f275145adfd..da8e7f03abd2 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp @@ -349,16 +349,23 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo, } } -void ARMInstPrinter::printOperand(const MCInst *MI, uint64_t Address, - unsigned OpNum, const MCSubtargetInfo &STI, - raw_ostream &O) { +void ARMInstPrinter::printPCRelativeOperand(const MCInst *MI, uint64_t Address, + unsigned OpNum, + const MCSubtargetInfo &STI, + raw_ostream &O) { const MCOperand &Op = MI->getOperand(OpNum); - if (!Op.isImm() || !PrintBranchImmAsAddress || getUseMarkup()) + if (!Op.isImm() || getUseMarkup()) return printOperand(MI, OpNum, STI, O); + + if (!PrintBranchImmAsAddress) { + O << markup(""); + return; + } + uint64_t Target = ARM_MC::evaluateBranchTarget(MII.get(MI->getOpcode()), Address, Op.getImm()); Target &= 0xffffffff; - O << formatHex(Target); + O << markup(""); if (CommentStream) *CommentStream << "imm = #" << formatImm(Op.getImm()) << '\n'; } @@ -372,7 +379,7 @@ void ARMInstPrinter::printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum, return; } - O << markup("") << ", "; int32_t OffImm = (int32_t)MO1.getImm(); bool isSub = OffImm < 0; @@ -381,9 +388,9 @@ void ARMInstPrinter::printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum, if (OffImm == INT32_MIN) OffImm = 0; if (isSub) { - O << markup(""); + O << markup(""); } else { - O << markup(""); + O << markup(""); } O << "]" << markup(">"); } @@ -1171,6 +1178,10 @@ void ARMInstPrinter::printT2SOOperand(const MCInst *MI, unsigned OpNum, ARM_AM::getSORegOffset(MO2.getImm()), UseMarkup); } +StringRef ARMInstPrinter::immediateOffsetMarkup(const MCOperand &Operand) { + return markup(Operand.getReg() == ARM::PC ? " void ARMInstPrinter::printAddrModeImm12Operand(const MCInst *MI, unsigned OpNum, const MCSubtargetInfo &STI, @@ -1191,11 +1202,15 @@ void ARMInstPrinter::printAddrModeImm12Operand(const MCInst *MI, unsigned OpNum, // Special value for #-0. All others are normal. if (OffImm == INT32_MIN) OffImm = 0; + if (isSub) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#-" << formatImm(-OffImm) << markup(">"); } else if (AlwaysPrintImm0 || OffImm > 0) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#" << formatImm(OffImm) << markup(">"); } + O << "]" << markup(">"); } @@ -1215,11 +1230,15 @@ void ARMInstPrinter::printT2AddrModeImm8Operand(const MCInst *MI, // Don't print +0. if (OffImm == INT32_MIN) OffImm = 0; + if (isSub) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#-" << -OffImm << markup(">"); } else if (AlwaysPrintImm0 || OffImm > 0) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#" << OffImm << markup(">"); } + O << "]" << markup(">"); } @@ -1247,11 +1266,15 @@ void ARMInstPrinter::printT2AddrModeImm8s4Operand(const MCInst *MI, // Don't print +0. if (OffImm == INT32_MIN) OffImm = 0; + if (isSub) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#-" << -OffImm << markup(">"); } else if (AlwaysPrintImm0 || OffImm > 0) { - O << ", " << markup(""); + O << ", " << immediateOffsetMarkup(MO1); + O << "#" << OffImm << markup(">"); } + O << "]" << markup(">"); } @@ -1264,8 +1287,8 @@ void ARMInstPrinter::printT2AddrModeImm0_1020s4Operand( O << markup(""); + O << ", " << markup(MO1.getReg() == ARM::PC ? ""); } O << "]" << markup(">"); } @@ -1275,7 +1298,7 @@ void ARMInstPrinter::printT2AddrModeImm8OffsetOperand( raw_ostream &O) { const MCOperand &MO1 = MI->getOperand(OpNum); int32_t OffImm = (int32_t)MO1.getImm(); - O << ", " << markup(""); + O << markup(""); else - O << markup(""); + O << markup(""); } void MipsInstPrinter::printBranchOperand(const MCInst *MI, uint64_t Address, @@ -168,9 +168,9 @@ void MipsInstPrinter::printBranchOperand(const MCInst *MI, uint64_t Address, Target &= 0xffffffff; else if (STI.hasFeature(Mips::FeatureMips16)) Target &= 0xffff; - O << markup(""); + O << markup(""); } else { - O << markup(""); + O << markup(""); } } diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp index 3e0e385b25c4..8c72675ed8e3 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp @@ -165,7 +165,7 @@ void SystemZInstPrinter::printPCRelOperand(const MCInst *MI, int OpNum, raw_ostream &O) { const MCOperand &MO = MI->getOperand(OpNum); if (MO.isImm()) { - O << markup(""); } else diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp index 5a1c4ec81e1b..82255dade9d0 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp @@ -434,8 +434,14 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op, if (DispSpec.isImm()) { int64_t DispVal = DispSpec.getImm(); - if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) - O << formatImm(DispVal); + if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) { + bool IsRIPBased = BaseReg.getReg() == X86::RIP || + BaseReg.getReg() == X86::EIP; + if (IsRIPBased && !IndexReg.getReg()) + O << markup(""); + else + O << markup(""); + } } else { assert(DispSpec.isExpr() && "non-immediate displacement for LEA?"); DispSpec.getExpr()->print(O, &MAI); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp index f2cb3dfc8ed0..abb2ebb642f7 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp @@ -322,7 +322,7 @@ void X86InstPrinterCommon::printPCRelImm(const MCInst *MI, uint64_t Address, const MCOperand &Op = MI->getOperand(OpNo); if (Op.isImm()) { - O << markup("(Op.getExpr()); int64_t Address; if (BranchTarget && BranchTarget->evaluateAsAbsolute(Address)) { - O << markup(""); + O << markup(""); } else { // Otherwise, just print the expression. Op.getExpr()->print(O, &MAI); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp index 84da39c4a295..9a8d8f14be2f 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp @@ -419,7 +419,13 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op, DispVal = -DispVal; } } - O << markup(""); + + bool IsRIPBased = BaseReg.getReg() == X86::RIP || + BaseReg.getReg() == X86::EIP; + if (IsRIPBased && !IndexReg.getReg()) + O << markup(""); + else + O << markup(""); } } diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp index a68035989a93..789839f3d418 100644 --- a/llvm/lib/TargetParser/Triple.cpp +++ b/llvm/lib/TargetParser/Triple.cpp @@ -352,6 +352,7 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) { .Case("x86", x86) .Case("i386", x86) .Case("x86-64", x86_64) + .Case("x86_64", x86_64) .Case("xcore", xcore) .Case("nvptx", nvptx) .Case("nvptx64", nvptx64) diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 42158e4e05dd..5cb037925efa 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -3494,7 +3494,7 @@ template struct CachedReachabilityAA : public BaseTy { using RQITy = ReachabilityQueryInfo; - CachedReachabilityAA(const IRPosition &IRP, Attributor &A) + CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {} /// See AbstractAttribute::isQueryAA. diff --git a/llvm/lib/Transforms/IPO/SCCP.cpp b/llvm/lib/Transforms/IPO/SCCP.cpp index 5c1582ddfdae..b42904ab65ad 100644 --- a/llvm/lib/Transforms/IPO/SCCP.cpp +++ b/llvm/lib/Transforms/IPO/SCCP.cpp @@ -114,11 +114,15 @@ static bool runIPSCCP( function_ref getAnalysis, bool IsFuncSpecEnabled) { SCCPSolver Solver(DL, GetTLI, M.getContext()); + auto T = make_task_on_set(make_address_range(M), "LLVM SCC Passes"); + FunctionSpecializer Specializer(Solver, M, FAM, GetTLI, GetTTI, GetAC); // Loop over all functions, marking arguments to those with their addresses // taken or that are external as overdefined. for (Function &F : M) { + T.advance(&F, F.getName()); + if (F.isDeclaration()) continue; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index 41bc65620ff6..0fc30d643e83 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -374,6 +374,7 @@ void PointerReplacer::replace(Instruction *I) { Type *NewTy = getReplacement(PHI->getIncomingValue(0))->getType(); auto *NewPHI = PHINode::Create(NewTy, PHI->getNumIncomingValues(), PHI->getName(), PHI); + NewPHI->setDebugLoc(PHI->getDebugLoc()); for (unsigned int I = 0; I < PHI->getNumIncomingValues(); ++I) NewPHI->addIncoming(getReplacement(PHI->getIncomingValue(I)), PHI->getIncomingBlock(I)); @@ -768,14 +769,7 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) { } if (auto *AT = dyn_cast(T)) { - auto *ET = AT->getElementType(); auto NumElements = AT->getNumElements(); - if (NumElements == 1) { - LoadInst *NewLoad = IC.combineLoadToNewType(LI, ET, ".unpack"); - NewLoad->setAAMetadata(LI.getAAMetadata()); - return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue( - PoisonValue::get(T), NewLoad, 0, Name)); - } // Bail out if the array is too large. Ideally we would like to optimize // arrays of arbitrary size but this has a terrible impact on compile time. @@ -784,6 +778,14 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) { if (NumElements > IC.MaxArraySizeForCombine) return nullptr; + auto *ET = AT->getElementType(); + if (NumElements == 1) { + LoadInst *NewLoad = IC.combineLoadToNewType(LI, ET, ".unpack"); + NewLoad->setAAMetadata(LI.getAAMetadata()); + return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue( + PoisonValue::get(T), NewLoad, 0, Name)); + } + const DataLayout &DL = IC.getDataLayout(); auto EltSize = DL.getTypeAllocSize(ET); const auto Align = LI.getAlign(); @@ -1291,13 +1293,7 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) { } if (auto *AT = dyn_cast(T)) { - // If the array only have one element, we unpack. auto NumElements = AT->getNumElements(); - if (NumElements == 1) { - V = IC.Builder.CreateExtractValue(V, 0); - combineStoreToNewValue(IC, SI, V); - return true; - } // Bail out if the array is too large. Ideally we would like to optimize // arrays of arbitrary size but this has a terrible impact on compile time. @@ -1306,6 +1302,12 @@ static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) { if (NumElements > IC.MaxArraySizeForCombine) return false; + // If the array only have one element, we unpack. + if (NumElements == 1) { + V = IC.Builder.CreateExtractValue(V, 0); + combineStoreToNewValue(IC, SI, V); + return true; + } const DataLayout &DL = IC.getDataLayout(); auto EltSize = DL.getTypeAllocSize(AT->getElementType()); const auto Align = SI.getAlign(); diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 7f59729f0085..b3cb2d4167ed 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -450,6 +450,7 @@ Instruction *InstCombinerImpl::foldPHIArgBinOpIntoPHI(PHINode &PN) { NewLHS = PHINode::Create(LHSType, PN.getNumIncomingValues(), FirstInst->getOperand(0)->getName() + ".pn"); NewLHS->addIncoming(InLHS, PN.getIncomingBlock(0)); + NewLHS->setDebugLoc(PN.getDebugLoc()); InsertNewInstBefore(NewLHS, PN); LHSVal = NewLHS; } @@ -458,6 +459,7 @@ Instruction *InstCombinerImpl::foldPHIArgBinOpIntoPHI(PHINode &PN) { NewRHS = PHINode::Create(RHSType, PN.getNumIncomingValues(), FirstInst->getOperand(1)->getName() + ".pn"); NewRHS->addIncoming(InRHS, PN.getIncomingBlock(0)); + NewRHS->setDebugLoc(PN.getDebugLoc()); InsertNewInstBefore(NewRHS, PN); RHSVal = NewRHS; } @@ -728,6 +730,7 @@ Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { PHINode *NewPN = PHINode::Create(FirstLI->getOperand(0)->getType(), PN.getNumIncomingValues(), PN.getName()+".in"); + NewPN->setDebugLoc(PN.getDebugLoc()); Value *InVal = FirstLI->getOperand(0); NewPN->addIncoming(InVal, PN.getIncomingBlock(0)); @@ -851,6 +854,7 @@ Instruction *InstCombinerImpl::foldPHIArgZextsIntoPHI(PHINode &Phi) { Phi.getName() + ".shrunk"); for (unsigned I = 0; I != NumIncomingValues; ++I) NewPhi->addIncoming(NewIncoming[I], Phi.getIncomingBlock(I)); + NewPhi->setDebugLoc(Phi.getDebugLoc()); InsertNewInstBefore(NewPhi, Phi); return CastInst::CreateZExtOrBitCast(NewPhi, Phi.getType()); @@ -921,6 +925,7 @@ Instruction *InstCombinerImpl::foldPHIArgOpIntoPHI(PHINode &PN) { PHINode *NewPN = PHINode::Create(FirstInst->getOperand(0)->getType(), PN.getNumIncomingValues(), PN.getName()+".in"); + NewPN->setDebugLoc(PN.getDebugLoc()); Value *InVal = FirstInst->getOperand(0); NewPN->addIncoming(InVal, PN.getIncomingBlock(0)); diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index fb6f4f96ea48..5a5d6a9cab5b 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -135,6 +135,7 @@ static constexpr unsigned InstCombineDefaultInfiniteLoopThreshold = 100; #else static constexpr unsigned InstCombineDefaultInfiniteLoopThreshold = 1000; #endif +static constexpr unsigned InstCombineDefaultMaxArraySize = 1024; static cl::opt EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"), @@ -155,9 +156,10 @@ static cl::opt InfiniteLoopDetectionThreshold( "infinite loop"), cl::init(InstCombineDefaultInfiniteLoopThreshold), cl::Hidden); -static cl::opt -MaxArraySize("instcombine-maxarray-size", cl::init(1024), - cl::desc("Maximum array size considered when doing a combine")); +cl::opt MaxArraySize( + "instcombine-maxarray-size", + cl::desc("Maximum array size considered when doing a combine"), + cl::init(InstCombineDefaultMaxArraySize)); // FIXME: Remove this flag when it is no longer necessary to convert // llvm.dbg.declare to avoid inaccurate debug info. Setting this to false @@ -1263,6 +1265,7 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) { PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues()); InsertNewInstBefore(NewPN, *PN); NewPN->takeName(PN); + NewPN->setDebugLoc(PN->getDebugLoc()); // If we are going to have to insert a new computation, do so right before the // predecessor's terminator. @@ -1291,6 +1294,10 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) { replaceInstUsesWith(*User, NewPN); eraseInstFromFunction(*User); } + + replaceAllDbgUsesWith(const_cast(*PN), + const_cast(*NewPN), + const_cast(*PN), DT); return replaceInstUsesWith(I, NewPN); } @@ -4609,12 +4616,16 @@ static bool combineInstructionsOverFunction( RemoveRedundantDbgInstrs(&BB); } + Task T({}, "InstCombine"); + // Iterate while there is work to do. unsigned Iteration = 0; while (true) { ++NumWorklistIterations; ++Iteration; + T.advance(Twine("Iteration: ") + Twine(Iteration)); + if (Iteration > InfiniteLoopDetectionThreshold) { report_fatal_error( "Instruction Combining seems stuck in an infinite loop after " + diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 2865dece8723..fe73f418f782 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -75,6 +75,7 @@ #include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Scalar.h" @@ -1465,6 +1466,7 @@ static Instruction *cloneInstructionInExitBlock( PHINode *OpPN = PHINode::Create(OInst->getType(), PN.getNumIncomingValues(), OInst->getName() + ".lcssa", &ExitBlock.front()); + OpPN->setDebugLoc(OInst->getDebugLoc()); for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) OpPN->addIncoming(OInst, PN.getIncomingBlock(i)); Op = OpPN; @@ -1738,7 +1740,8 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop, // Move the new node to the destination block, before its terminator. moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE); - I.updateLocationAfterHoist(); + if (not EnableStrictDebugInformationPreservationStyle) + I.updateLocationAfterHoist(); if (isa(I)) ++NumMovedLoads; diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp index 7b396c6ee074..55012911016c 100644 --- a/llvm/lib/Transforms/Scalar/SCCP.cpp +++ b/llvm/lib/Transforms/Scalar/SCCP.cpp @@ -186,4 +186,3 @@ INITIALIZE_PASS_END(SCCPLegacyPass, "sccp", // createSCCPPass - This is the public interface to this file. FunctionPass *llvm::createSCCPPass() { return new SCCPLegacyPass(); } - diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 8339981e1bdc..50818a5209b7 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -118,6 +118,12 @@ STATISTIC(NumVectorized, "Number of vectorized aggregates"); /// GEPs. static cl::opt SROAStrictInbounds("sroa-strict-inbounds", cl::init(false), cl::Hidden); + +cl::opt +SROANoArrays("sroa-no-arrays", + cl::desc("Disables SROA for array aggregates"), + cl::init(false)); + namespace { /// Find linked dbg.assign and generate a new one with the correct /// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the @@ -882,7 +888,9 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { } void visitLoadInst(LoadInst &LI) { - assert((!LI.isSimple() || LI.getType()->isSingleValueType()) && + assert((!LI.isSimple() || + LI.getType()->isSingleValueType() || + (isa(LI.getType()) && SROANoArrays)) && "All simple FCA loads should have been pre-split"); if (!IsOffsetKnown) @@ -923,7 +931,9 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { return markAsDead(SI); } - assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) && + assert((!SI.isSimple() || + ValOp->getType()->isSingleValueType() || + (isa(ValOp->getType()) && SROANoArrays)) && "All simple FCA stores should have been pre-split"); handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); } @@ -3722,6 +3732,9 @@ class AggLoadStoreRewriter : public InstVisitor { if (!LI.isSimple() || LI.getType()->isSingleValueType()) return false; + if (isa(LI.getType()) and SROANoArrays) + return false; + // We have an aggregate being loaded, split it apart. LLVM_DEBUG(dbgs() << " original: " << LI << "\n"); LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(), @@ -3793,6 +3806,9 @@ class AggLoadStoreRewriter : public InstVisitor { if (V->getType()->isSingleValueType()) return false; + if (isa(V->getType()) and SROANoArrays) + return false; + // We have an aggregate being stored, split it apart. LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI, diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp index e014f5d1eb04..6fd4624ea564 100644 --- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp +++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp @@ -32,6 +32,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/CFG.h" #include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" @@ -108,6 +109,7 @@ performBlockTailMerging(Function &F, ArrayRef BBs, std::get<1>(I) = PHINode::Create(std::get<0>(I)->getType(), /*NumReservedValues=*/BBs.size(), CanonicalBB->getName() + ".op"); + std::get<1>(I)->setDebugLoc(Term->getDebugLoc()); std::get<1>(I)->insertInto(CanonicalBB, CanonicalBB->end()); } // Make it so that this canonical block actually has the right @@ -117,6 +119,8 @@ performBlockTailMerging(Function &F, ArrayRef BBs, // If the canonical terminator has operands, rewrite it to take PHI's. for (auto I : zip(NewOps, CanonicalTerm->operands())) std::get<1>(I) = std::get<0>(I); + + CanonicalTerm->setDebugLoc(Term->getDebugLoc()); } // Now, go through each block (with the current terminator type) @@ -142,8 +146,10 @@ performBlockTailMerging(Function &F, ArrayRef BBs, // And turn BB into a block that just unconditionally branches // to the canonical block. + Instruction *BI = BranchInst::Create(CanonicalBB, BB); + BI->setDebugLoc(Term->getDebugLoc()); Term->eraseFromParent(); - BranchInst::Create(CanonicalBB, BB); + if (Updates) Updates->push_back({DominatorTree::Insert, BB, CanonicalBB}); } diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index 58a226fc601c..a58282651199 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -856,6 +856,7 @@ void llvm::createPHIsForSplitLoopExit(ArrayRef Preds, SplitBB->isLandingPad() ? &SplitBB->front() : SplitBB->getTerminator()); for (BasicBlock *BB : Preds) NewPN->addIncoming(V, BB); + NewPN->setDebugLoc(PN.getDebugLoc()); // Update the original PHI. PN.setIncomingValue(Idx, NewPN); @@ -1156,6 +1157,7 @@ static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB, // Create the new PHI node, insert it into NewBB at the end of the block PHINode *NewPHI = PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI); + NewPHI->setDebugLoc(PN->getDebugLoc()); // NOTE! This loop walks backwards for a reason! First off, this minimizes // the cost of removal if we end up removing a large number of values, and diff --git a/llvm/lib/Transforms/Utils/CloneModule.cpp b/llvm/lib/Transforms/Utils/CloneModule.cpp index 55e051298a9a..f0f6de4c4b9f 100644 --- a/llvm/lib/Transforms/Utils/CloneModule.cpp +++ b/llvm/lib/Transforms/Utils/CloneModule.cpp @@ -49,6 +49,15 @@ std::unique_ptr llvm::CloneModule(const Module &M, std::unique_ptr llvm::CloneModule( const Module &M, ValueToValueMapTy &VMap, function_ref ShouldCloneDefinition) { + return CloneModule(M, VMap, [&ShouldCloneDefinition](const GlobalValue *GV) { + return ShouldCloneDefinition(GV) ? CloneAction::Clone + : CloneAction::MakeDeclaration; + }); +} + +std::unique_ptr +llvm::CloneModule(const Module &M, ValueToValueMapTy &VMap, + function_ref Action) { // First off, we need to create the new module. std::unique_ptr New = std::make_unique(M.getModuleIdentifier(), M.getContext()); @@ -57,11 +66,16 @@ std::unique_ptr llvm::CloneModule( New->setTargetTriple(M.getTargetTriple()); New->setModuleInlineAsm(M.getModuleInlineAsm()); + DenseMap Actions; // Loop over all of the global variables, making corresponding globals in the // new module. Here we add them to the VMap and to the new Module. We // don't worry about attributes or initializers, they will come later. // for (const GlobalVariable &I : M.globals()) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + GlobalVariable *NewGV = new GlobalVariable( *New, I.getValueType(), I.isConstant(), I.getLinkage(), (Constant *)nullptr, I.getName(), (GlobalVariable *)nullptr, @@ -72,6 +86,10 @@ std::unique_ptr llvm::CloneModule( // Loop over the functions in the module, making external functions as before for (const Function &I : M) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + Function *NF = Function::Create(cast(I.getValueType()), I.getLinkage(), I.getAddressSpace(), I.getName(), New.get()); @@ -81,7 +99,11 @@ std::unique_ptr llvm::CloneModule( // Loop over the aliases in the module for (const GlobalAlias &I : M.aliases()) { - if (!ShouldCloneDefinition(&I)) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + + if (Actions[&I] == CloneAction::MakeDeclaration) { // An alias cannot act as an external reference, so we need to create // either a function or a global variable depending on the value type. // FIXME: Once pointee types are gone we can probably pick one or the @@ -110,6 +132,10 @@ std::unique_ptr llvm::CloneModule( } for (const GlobalIFunc &I : M.ifuncs()) { + Actions[&I] = Action(&I); + if (Actions[&I] == CloneAction::Omit) + continue; + // Defer setting the resolver function until after functions are cloned. auto *GI = GlobalIFunc::create(I.getValueType(), I.getAddressSpace(), @@ -123,6 +149,9 @@ std::unique_ptr llvm::CloneModule( // over... We also set the attributes on the global now. // for (const GlobalVariable &G : M.globals()) { + if (Actions[&G] == CloneAction::Omit) + continue; + GlobalVariable *GV = cast(VMap[&G]); SmallVector, 1> MDs; @@ -133,7 +162,7 @@ std::unique_ptr llvm::CloneModule( if (G.isDeclaration()) continue; - if (!ShouldCloneDefinition(&G)) { + if (Actions[&G] == CloneAction::MakeDeclaration) { // Skip after setting the correct linkage for an external reference. GV->setLinkage(GlobalValue::ExternalLinkage); continue; @@ -147,6 +176,9 @@ std::unique_ptr llvm::CloneModule( // Similarly, copy over function bodies now... // for (const Function &I : M) { + if (Actions[&I] == CloneAction::Omit) + continue; + Function *F = cast(VMap[&I]); if (I.isDeclaration()) { @@ -159,7 +191,7 @@ std::unique_ptr llvm::CloneModule( continue; } - if (!ShouldCloneDefinition(&I)) { + if (Actions[&I] == CloneAction::MakeDeclaration) { // Skip after setting the correct linkage for an external reference. F->setLinkage(GlobalValue::ExternalLinkage); // Personality function is not valid on a declaration. @@ -186,14 +218,18 @@ std::unique_ptr llvm::CloneModule( // And aliases for (const GlobalAlias &I : M.aliases()) { // We already dealt with undefined aliases above. - if (!ShouldCloneDefinition(&I)) + if (Actions[&I] != CloneAction::Clone) continue; + GlobalAlias *GA = cast(VMap[&I]); if (const Constant *C = I.getAliasee()) GA->setAliasee(MapValue(C, VMap)); } for (const GlobalIFunc &I : M.ifuncs()) { + if (Actions[&I] == CloneAction::Omit) + continue; + GlobalIFunc *GI = cast(VMap[&I]); if (const Constant *Resolver = I.getResolver()) GI->setResolver(MapValue(Resolver, VMap)); diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index c1fe10504e45..b2df50f26594 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -231,8 +231,8 @@ buildExtractionBlockSet(ArrayRef BBs, DominatorTree *DT, if (!Result.count(PBB)) { LLVM_DEBUG(dbgs() << "No blocks in this region may have entries from " "outside the region except for the first block!\n" - << "Problematic source BB: " << BB->getName() << "\n" - << "Problematic destination BB: " << PBB->getName() + << "Problematic source BB: " << PBB->getName() << "\n" + << "Problematic destination BB: " << BB->getName() << "\n"); return {}; } diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index b2ed95b05e04..581264152aa3 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -3302,7 +3302,12 @@ bool llvm::recognizeBSwapOrBitReverseIdiom( BitProvenance = BitProvenance.drop_back(); if (BitProvenance.empty()) return false; // TODO - handle null value? - DemandedTy = Type::getIntNTy(I->getContext(), BitProvenance.size()); + + unsigned Size = BitProvenance.size(); + if (!I->getModule()->getDataLayout().isLegalInteger(Size)) + return false; + + DemandedTy = Type::getIntNTy(I->getContext(), Size); if (auto *IVecTy = dyn_cast(ITy)) DemandedTy = VectorType::get(DemandedTy, IVecTy); } diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp index 87a0e54e2704..19473c08b82a 100644 --- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp +++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp @@ -400,6 +400,7 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader, PHINode *PN = cast(I); PHINode *NewPN = PHINode::Create(PN->getType(), BackedgeBlocks.size(), PN->getName()+".be", BETerminator); + NewPN->setDebugLoc(PN->getDebugLoc()); // Loop over the PHI node, moving all entries except the one for the // preheader over to the new PHI node. diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp index 75ea9dc5dfc0..1a51a3ba3e44 100644 --- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -43,6 +43,7 @@ #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" #include @@ -959,6 +960,13 @@ bool PromoteMem2Reg::QueuePhiNode(BasicBlock *BB, unsigned AllocaNo, PN = PHINode::Create(Allocas[AllocaNo]->getAllocatedType(), getNumPreds(BB), Allocas[AllocaNo]->getName() + "." + Twine(Version++), &BB->front()); + + if (EnableStrictDebugInformationPreservationStyle) { + // Keep the debug information from the original alloca, to avoid discarding + // debug information. + PN->setDebugLoc(Allocas[AllocaNo]->getDebugLoc()); + } + ++NumPHIInsert; PhiToAllocaMap[PN] = AllocaNo; return true; @@ -968,6 +976,11 @@ bool PromoteMem2Reg::QueuePhiNode(BasicBlock *BB, unsigned AllocaNo, /// create a merged location incorporating \p DL, or to set \p DL directly. static void updateForIncomingValueLocation(PHINode *PN, DebugLoc DL, bool ApplyMergedLoc) { + if (EnableStrictDebugInformationPreservationStyle and not DL) { + // Do not override valid debug information with an empty location. + return; + } + if (ApplyMergedLoc) PN->applyMergedLocation(PN->getDebugLoc(), DL); else diff --git a/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/llvm/lib/Transforms/Utils/SSAUpdater.cpp index 2520aa5d9db0..2453ea24699c 100644 --- a/llvm/lib/Transforms/Utils/SSAUpdater.cpp +++ b/llvm/lib/Transforms/Utils/SSAUpdater.cpp @@ -27,6 +27,7 @@ #include "llvm/IR/Value.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/SSAUpdaterImpl.h" #include @@ -269,6 +270,13 @@ class SSAUpdaterTraits { SSAUpdater *Updater) { PHINode *PHI = PHINode::Create(Updater->ProtoType, NumPreds, Updater->ProtoName, &BB->front()); + + if (EnableStrictDebugInformationPreservationStyle) { + // This is not the best, the proper solution would be to accept the debug + // location from the callee, but this will do for now. + PHI->setDebugLoc(BB->getTerminator()->getDebugLoc()); + } + return PHI; } diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 24f1966edd37..df95b4d38729 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -21,6 +21,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/PatternMatch.h" @@ -1600,8 +1601,11 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) { // specified loop. BasicBlock *Header = L->getHeader(); pred_iterator HPB = pred_begin(Header), HPE = pred_end(Header); + + DebugLoc DebugLocation = Header->front().getDebugLoc(); CanonicalIV = PHINode::Create(Ty, std::distance(HPB, HPE), "indvar", &Header->front()); + CanonicalIV->setDebugLoc(DebugLocation); rememberInstruction(CanonicalIV); SmallSet PredSeen; diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 9e0483966d3e..d212a065abed 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -67,6 +67,7 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/DebugInfoPreservation.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" @@ -1105,12 +1106,14 @@ static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses( Instruction *NewBonusInst = BonusInst.clone(); - if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) { - // Unless the instruction has the same !dbg location as the original - // branch, drop it. When we fold the bonus instructions we want to make - // sure we reset their debug locations in order to avoid stepping on - // dead code caused by folding dead branches. - NewBonusInst->setDebugLoc(DebugLoc()); + if (not EnableStrictDebugInformationPreservationStyle) { + if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) { + // Unless the instruction has the same !dbg location as the original + // branch, drop it. When we fold the bonus instructions we want to make + // sure we reset their debug locations in order to avoid stepping on + // dead code caused by folding dead branches. + NewBonusInst->setDebugLoc(DebugLoc()); + } } RemapInstruction(NewBonusInst, VMap, @@ -1970,6 +1973,12 @@ static bool sinkLastInstruction(ArrayRef Blocks) { for (auto *I : Insts) PN->addIncoming(I->getOperand(O), I->getParent()); NewOperands.push_back(PN); + + // Merge all the incoming debug locations together. + PN->setDebugLoc(Insts.front()->getDebugLoc()); + for (Instruction *I : Insts) + if (I->getDebugLoc()) + PN->applyMergedLocation(PN->getDebugLoc(), I->getDebugLoc()); } // Arbitrarily use I0 as the new "common" instruction; remap its operands diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp index 4e83d2f6e3c6..7e8b704f0b87 100644 --- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -1118,8 +1118,8 @@ class WidenIV { /// common dominator for the incoming blocks. A nullptr can be returned if no /// viable location is found: it may happen if User is a PHI and Def only comes /// to this PHI from unreachable blocks. -static Instruction *getInsertPointForUses(Instruction *User, Value *Def, - DominatorTree *DT, LoopInfo *LI) { +Instruction *llvm::getInsertPointForUses(Instruction *User, Value *Def, + DominatorTree *DT, LoopInfo *LI) { PHINode *PHI = dyn_cast(User); if (!PHI) return User; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index d554f438c804..5e82a3829868 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -993,7 +993,7 @@ void VPlanIngredient::print(raw_ostream &O) const { #endif -template void DomTreeBuilder::Calculate(VPDominatorTree &DT); +template void DomTreeBuilder::Calculate(VPDominatorTree &DT); void VPValue::replaceAllUsesWith(VPValue *New) { for (unsigned J = 0; J < getNumUsers();) { diff --git a/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir b/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir index 81c2b99fc068..e880c03c1a17 100644 --- a/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir +++ b/llvm/test/CodeGen/MIR/X86/variable-sized-stack-objects.mir @@ -30,11 +30,11 @@ frameInfo: # CHECK-NEXT: - { id: 1, name: '', type: default, offset: -32, size: 8, alignment: 8, # CHECK-NEXT: stack-id: default, callee-saved-register: '', callee-saved-restored: true, # CHECK-NEXT: debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } -# CHECK-NEXT: - { id: 2, name: y, type: variable-sized, offset: -32, alignment: 1, +# CHECK-NEXT: - { id: 2, name: 'y', type: variable-sized, offset: -32, alignment: 1, stack: - { id: 0, offset: -20, size: 4, alignment: 4 } - { id: 1, offset: -32, size: 8, alignment: 8 } - - { id: 2, name: y, type: variable-sized, offset: -32, alignment: 1 } + - { id: 2, name: 'y', type: variable-sized, offset: -32, alignment: 1 } body: | bb.0.entry: MOV32mr $rsp, 1, _, -4, _, $edi diff --git a/llvm/test/DebugInfo/COFF/array-odr-violation.ll b/llvm/test/DebugInfo/COFF/array-odr-violation.ll index 0283515c269b..3a1f8c852331 100644 --- a/llvm/test/DebugInfo/COFF/array-odr-violation.ll +++ b/llvm/test/DebugInfo/COFF/array-odr-violation.ll @@ -17,21 +17,27 @@ ; FIXME: sizeof(a) in the user program is 1, but we claim it is 4 because ; sometimes the frontend lies to us. See array-types-advanced.ll for an example. ; -; CHECK: # Array (0x1004) +; CHECK: # Alias (0x1004) +; CHECK: .short 0xe # Record length +; CHECK: .short 0x150a # Record kind: LF_ALIAS +; CHECK: .long 0x1003 # UnderlyingType: YYSTYPE +; CHECK: .asciz "YYSTYPE" # Name + +; CHECK: # Array (0x1005) ; CHECK: .short 0xe # Record length ; CHECK: .short 0x1503 # Record kind: LF_ARRAY -; CHECK: .long 0x1003 # ElementType: YYSTYPE +; CHECK: .long 0x1004 # ElementType: YYSTYPE ; CHECK: .long 0x23 # IndexType: unsigned __int64 ; CHECK: .short 0x4 # SizeOf ; CHECK: .byte 0 # Name ; CHECK: .byte 241 -; CHECK: # Union (0x1006) +; CHECK: # Union (0x1007) ; CHECK: .short 0x22 # Record length ; CHECK: .short 0x1506 # Record kind: LF_UNION ; CHECK: .short 0x1 # MemberCount ; CHECK: .short 0x600 # Properties ( HasUniqueName (0x200) | Sealed (0x400) ) -; CHECK: .long 0x1005 # FieldList: +; CHECK: .long 0x1006 # FieldList: ; CHECK: .short 0x4 # SizeOf ; CHECK: .asciz "YYSTYPE" # Name ; CHECK: .asciz ".?ATYYSTYPE@@" # LinkageName diff --git a/llvm/test/DebugInfo/COFF/global-type-hashes.ll b/llvm/test/DebugInfo/COFF/global-type-hashes.ll index 586b97d8fd19..50ad9efb9953 100644 --- a/llvm/test/DebugInfo/COFF/global-type-hashes.ll +++ b/llvm/test/DebugInfo/COFF/global-type-hashes.ll @@ -240,7 +240,7 @@ attributes #3 = { mustprogress noinline nounwind optnone "frame-pointer"="none" ; YAML: Attrs: 3 ; YAML: Type: 116 ; YAML: FieldOffset: 4 -; YAML: Name: Y +; YAML: Name: 'Y' ; YAML: - Kind: LF_ONEMETHOD ; YAML: OneMethod: ; YAML: Type: 4103 diff --git a/llvm/test/DebugInfo/COFF/global_rust.ll b/llvm/test/DebugInfo/COFF/global_rust.ll index 1bbfe0f49623..e107771b58fe 100644 --- a/llvm/test/DebugInfo/COFF/global_rust.ll +++ b/llvm/test/DebugInfo/COFF/global_rust.ll @@ -29,7 +29,7 @@ ; CHECK: DataSym { ; CHECK: Kind: S_LDATA32 (0x110C) ; CHECK: DataOffset: .rdata+0x8 -; CHECK: Type: impl$::vtable_type$ (0x101D) +; CHECK: Type: impl$::vtable_type$ (0x1020) ; CHECK: DisplayName: impl$::vtable$ ; CHECK: LinkageName: .rdata ; CHECK: } diff --git a/llvm/test/DebugInfo/COFF/int8-char-type.ll b/llvm/test/DebugInfo/COFF/int8-char-type.ll index 3dd5aae5421b..6e3afae5cd7a 100644 --- a/llvm/test/DebugInfo/COFF/int8-char-type.ll +++ b/llvm/test/DebugInfo/COFF/int8-char-type.ll @@ -5,10 +5,21 @@ ; DW_ATE_[un]signed encoding for all integer types if they don't have distinct ; integer types for characters types. This was PR30552. +; CHECK: Alias (0x1000) { +; CHECK-NEXT: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK-NEXT: UnderlyingType: signed char (0x10) +; CHECK-NEXT: Name: int8_t +; CHECK-NEXT: } +; CHECK: Alias (0x1001) { +; CHECK-NEXT: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK-NEXT: UnderlyingType: unsigned char (0x20) +; CHECK-NEXT: Name: uint8_t +; CHECK-NEXT: } + ; CHECK-LABEL: GlobalData { ; CHECK-NEXT: Kind: S_GDATA32 (0x110D) ; CHECK-NEXT: DataOffset: -; CHECK-NEXT: Type: signed char (0x10) +; CHECK-NEXT: Type: int8_t (0x1000) ; CHECK-NEXT: DisplayName: x ; CHECK-NEXT: LinkageName: x ; CHECK-NEXT: } @@ -16,7 +27,7 @@ ; CHECK-LABEL: GlobalData { ; CHECK-NEXT: Kind: S_GDATA32 (0x110D) ; CHECK-NEXT: DataOffset: -; CHECK-NEXT: Type: unsigned char (0x20) +; CHECK-NEXT: Type: uint8_t (0x1001) ; CHECK-NEXT: DisplayName: y ; CHECK-NEXT: LinkageName: y ; CHECK-NEXT: } diff --git a/llvm/test/DebugInfo/COFF/integer-128.ll b/llvm/test/DebugInfo/COFF/integer-128.ll index 57ebdcc144af..e29e37388544 100644 --- a/llvm/test/DebugInfo/COFF/integer-128.ll +++ b/llvm/test/DebugInfo/COFF/integer-128.ll @@ -18,7 +18,7 @@ ; ASM-LABEL: .long 241 # Symbol subsection for globals ; ; ASM-LABEL: .short 4359 # Record kind: S_CONSTANT -; ASM-NEXT: .long 4110 # Type +; ASM-NEXT: .long 4111 # Type ; ASM-NEXT: .byte 0x0a, 0x80, 0xff, 0xff # Value ; ASM-NEXT: .byte 0xff, 0xff, 0xff, 0xff ; ASM-NEXT: .byte 0xff, 0xff @@ -26,7 +26,7 @@ ; ASM-NEXT: .p2align 2 ; ; ASM-LABEL: .short 4359 # Record kind: S_CONSTANT -; ASM-NEXT: .long 4111 # Type +; ASM-NEXT: .long 4112 # Type ; ASM-NEXT: .byte 0x09, 0x80, 0x00, 0x00 # Value ; ASM-NEXT: .byte 0x00, 0x00, 0x00, 0x00 ; ASM-NEXT: .byte 0x00, 0x80 diff --git a/llvm/test/DebugInfo/COFF/nested-types.ll b/llvm/test/DebugInfo/COFF/nested-types.ll index c68e4314925a..276054db6545 100644 --- a/llvm/test/DebugInfo/COFF/nested-types.ll +++ b/llvm/test/DebugInfo/COFF/nested-types.ll @@ -72,6 +72,12 @@ target triple = "x86_64-pc-windows-msvc19.0.24215" ; CHECK-NEXT: LinkageName: .?AW4InnerEnum@HasNested@@ ; CHECK-NEXT: } +; CHECK: Alias ([[INNERALIAS:0x.*]]) { +; CHECK-NEXT: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK-NEXT: UnderlyingType: int (0x74) +; CHECK-NEXT: Name: InnerTypedef +; CHECK-NEXT: } + ; CHECK: FieldList ([[UNNAMED_MEMBERS:0x.*]]) { ; CHECK-NEXT: TypeLeafKind: LF_FIELDLIST (0x1203) ; CHECK-NEXT: Enumerator { @@ -120,7 +126,7 @@ target triple = "x86_64-pc-windows-msvc19.0.24215" ; CHECK-NEXT: } ; CHECK-NEXT: NestedType { ; CHECK-NEXT: TypeLeafKind: LF_NESTTYPE (0x1510) -; CHECK-NEXT: Type: int (0x74) +; CHECK-NEXT: Type: InnerTypedef ([[INNERALIAS]]) ; CHECK-NEXT: Name: InnerTypedef ; CHECK-NEXT: } ; CHECK-NEXT: NestedType { diff --git a/llvm/test/DebugInfo/COFF/typedef.ll b/llvm/test/DebugInfo/COFF/typedef.ll index 892bd4dead0a..9b15beea5786 100644 --- a/llvm/test/DebugInfo/COFF/typedef.ll +++ b/llvm/test/DebugInfo/COFF/typedef.ll @@ -1,9 +1,16 @@ ; RUN: llc < %s -filetype=obj | llvm-readobj - --codeview | FileCheck %s +; CHECK: CodeViewTypes [ +; CHECK: Alias (0x1003) { +; CHECK: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK: UnderlyingType: wchar_t (0x71) +; CHECK: Name: XYZ +; CHECK: } +; CHECK: ] ; CHECK: CodeViewDebugInfo [ ; CHECK: Subsection [ ; CHECK: LocalSym { -; CHECK: Type: wchar_t (0x71) +; CHECK: Type: XYZ (0x1003) ; CHECK: Flags [ (0x0) ; CHECK: ] ; CHECK: VarName: foo @@ -11,7 +18,7 @@ ; CHECK: Subsection [ ; CHECK: SubSectionType: Symbols (0xF1) ; CHECK: UDTSym { -; CHECK: Type: wchar_t (0x71) +; CHECK: Type: XYZ (0x1003) ; CHECK: UDTName: XYZ ; CHECK: } ; CHECK: ] diff --git a/llvm/test/DebugInfo/COFF/types-array-advanced.ll b/llvm/test/DebugInfo/COFF/types-array-advanced.ll index f44510813372..eba2963d0001 100644 --- a/llvm/test/DebugInfo/COFF/types-array-advanced.ll +++ b/llvm/test/DebugInfo/COFF/types-array-advanced.ll @@ -157,12 +157,17 @@ ; CHECK: Volatile (0x2) ; CHECK: ] ; CHECK: } -; CHECK: Array (0x1010) { +; CHECK: Alias (0x1010) { +; CHECK: TypeLeafKind: LF_ALIAS (0x150A) +; CHECK: UnderlyingType: const volatile int (0x100F) +; CHECK: Name: T_INT +; CHECK: } +; CHECK: Array (0x1011) { ; CHECK: TypeLeafKind: LF_ARRAY (0x1503) -; CHECK: ElementType: const volatile int (0x100F) +; CHECK: ElementType: T_INT (0x1010) ; CHECK: IndexType: unsigned long (0x22) ; CHECK: SizeOf: 16 -; CHECK: Name: +; CHECK: Name: ; CHECK: } ; CHECK: ] diff --git a/llvm/test/DebugInfo/COFF/types-cvarargs.ll b/llvm/test/DebugInfo/COFF/types-cvarargs.ll index 68c7a7740b71..fd4b2048928a 100644 --- a/llvm/test/DebugInfo/COFF/types-cvarargs.ll +++ b/llvm/test/DebugInfo/COFF/types-cvarargs.ll @@ -34,7 +34,7 @@ ; CHECK: } ; CHECK: UDTSym { ; CHECK: Kind: S_UDT (0x1108) -; CHECK: Type: void (int, float, )* (0x100F) +; CHECK: Type: FuncTypedef (0x1010) ; CHECK: UDTName: FuncTypedef ; CHECK: } ; CHECK: ] diff --git a/llvm/test/DebugInfo/COFF/udts-complete.ll b/llvm/test/DebugInfo/COFF/udts-complete.ll index e5ee8521ab48..28cc6d81b1d7 100644 --- a/llvm/test/DebugInfo/COFF/udts-complete.ll +++ b/llvm/test/DebugInfo/COFF/udts-complete.ll @@ -1,8 +1,8 @@ ; RUN: llc -filetype=obj %s -o %t.obj ; RUN: llvm-pdbutil dump -symbols -types %t.obj | FileCheck %s -; Make sure that `gv`, `Bar`, and `Baz` all point to the complete type index of -; Foo (0x1002), and not the forward declaration index (0x1000). +; Make sure that `gv`, `Bar`, and `Baz` reference proper type indices. +; With LF_ALIAS, typedefs get their own type records. ; C++ source: ; struct Foo { @@ -19,26 +19,28 @@ ; CHECK: unique name: `.?AUFoo@@` ; CHECK: vtable: , base list: , field list: ; CHECK: options: forward ref | has unique name, sizeof 0 -; CHECK: 0x1001 | LF_FIELDLIST [size = 16] +; CHECK: 0x1001 | LF_ALIAS [size = 12] alias = Bar, underlying type = 0x1000 +; CHECK: 0x1002 | LF_ALIAS [size = 12] alias = Baz, underlying type = 0x1001 +; CHECK: 0x1003 | LF_FIELDLIST [size = 16] ; CHECK: - LF_MEMBER [name = `x`, Type = 0x0074 (int), offset = 0, attrs = public] -; CHECK: 0x1002 | LF_STRUCTURE [size = 36] `Foo` +; CHECK: 0x1004 | LF_STRUCTURE [size = 36] `Foo` ; CHECK: unique name: `.?AUFoo@@` -; CHECK: vtable: , base list: , field list: 0x1001 +; CHECK: vtable: , base list: , field list: 0x1003 ; CHECK: options: has unique name, sizeof 4 -; CHECK: 0x1004 | LF_UDT_SRC_LINE [size = 16] -; CHECK: udt = 0x1002, file = 4099, line = 1 +; CHECK: 0x1006 | LF_UDT_SRC_LINE [size = 16] +; CHECK: udt = 0x1004, file = 4101, line = 1 ; CHECK: Symbols ; CHECK: ============================================================ ; CHECK: Mod 0000 | `.debug$S`: ; CHECK: 0 | S_GDATA32 [size = 20] `gv` -; CHECK: type = 0x1002 (Foo), addr = 0000:0000 +; CHECK: type = 0x1002 (Baz), addr = 0000:0000 ; CHECK: 0 | S_UDT [size = 12] `Bar` -; CHECK: original type = 0x1002 +; CHECK: original type = 0x1001 ; CHECK: 0 | S_UDT [size = 12] `Baz` ; CHECK: original type = 0x1002 ; CHECK: 0 | S_UDT [size = 12] `Foo` -; CHECK: original type = 0x1002 +; CHECK: original type = 0x1004 ; ModuleID = 't.cpp' source_filename = "t.cpp" diff --git a/llvm/test/DebugInfo/COFF/udts.ll b/llvm/test/DebugInfo/COFF/udts.ll index 86d2c7a6c079..3dc752f5ed83 100644 --- a/llvm/test/DebugInfo/COFF/udts.ll +++ b/llvm/test/DebugInfo/COFF/udts.ll @@ -36,7 +36,7 @@ ; READOBJ: } ; READOBJ: UDTSym { ; READOBJ-NEXT: Kind: S_UDT (0x1108) -; READOBJ-NEXT: Type: int (0x74) +; READOBJ-NEXT: Type: FOO (0x{{[0-9A-F]+}}) ; READOBJ-NEXT: UDTName: f::FOO ; READOBJ-NEXT: } ; READOBJ-NEXT: ProcEnd { @@ -79,13 +79,13 @@ ; PDBUTIL: Symbols ; PDBUTIL-NEXT: ============================================================ ; PDBUTIL-NOT: S_UDT {{.*}} `A::C` -; PDBUTIL: S_UDT [size = 16] `f::FOO` -; PDBUTIL: S_UDT [size = 16] `g::pun` -; PDBUTIL: S_UDT [size = 12] `S` -; PDBUTIL: S_UDT [size = 12] `A` -; PDBUTIL: S_UDT [size = 16] `A::D` -; PDBUTIL: S_UDT [size = 12] `U` -; PDBUTIL: S_UDT [size = 12] `U` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `f::FOO` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `g::pun` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `S` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `A` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `A::D` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `U` +; PDBUTIL: S_UDT [size = {{[0-9]+}}] `U` source_filename = "test/DebugInfo/COFF/udts.ll" target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" diff --git a/llvm/test/DebugInfo/PDB/Inputs/typedef.ll b/llvm/test/DebugInfo/PDB/Inputs/typedef.ll new file mode 100644 index 000000000000..7c5b7bec7828 --- /dev/null +++ b/llvm/test/DebugInfo/PDB/Inputs/typedef.ll @@ -0,0 +1,47 @@ +; ModuleID = 'typedef.cpp' +source_filename = "typedef.cpp" +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.44.35214" + +; Function Attrs: mustprogress noinline norecurse nounwind optnone uwtable +define dso_local noundef i32 @main() #0 !dbg !14 { +entry: + %retval = alloca i32, align 4 + %val = alloca i8, align 1 + %val2 = alloca i64, align 8 + store i32 0, ptr %retval, align 4 + call void @llvm.dbg.declare(metadata ptr %val, metadata !19, metadata !DIExpression()), !dbg !22 + store i8 15, ptr %val, align 1, !dbg !22 + call void @llvm.dbg.declare(metadata ptr %val2, metadata !23, metadata !DIExpression()), !dbg !26 + store i64 -1, ptr %val2, align 8, !dbg !26 + ret i32 0, !dbg !27 +} + +; Function Attrs: nounwind readnone speculatable willreturn +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +attributes #0 = { mustprogress noinline norecurse nounwind optnone uwtable } +attributes #1 = { nounwind readnone speculatable willreturn } + +!llvm.dbg.cu = !{!2} +!llvm.module.flags = !{!7, !8} +!llvm.ident = !{!13} + +!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !5, producer: "clang version 22.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!5 = !DIFile(filename: "typedef.cpp", directory: "llvm-project") +!7 = !{i32 2, !"CodeView", i32 1} +!8 = !{i32 2, !"Debug Info Version", i32 3} +!13 = !{!"clang version 22.0.0git"} +!14 = distinct !DISubprogram(name: "main", scope: !5, file: !5, line: 6, type: !15, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2) +!15 = !DISubroutineType(types: !16) +!16 = !{!17} +!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!19 = !DILocalVariable(name: "val", scope: !14, file: !5, line: 7, type: !20) +!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "u8", file: !5, line: 3, baseType: !21) +!21 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char) +!22 = !DILocation(line: 7, scope: !14) +!23 = !DILocalVariable(name: "val2", scope: !14, file: !5, line: 8, type: !24) +!24 = !DIDerivedType(tag: DW_TAG_typedef, name: "i64", file: !5, line: 4, baseType: !25) +!25 = !DIBasicType(name: "long long", size: 64, encoding: DW_ATE_signed) +!26 = !DILocation(line: 8, scope: !14) +!27 = !DILocation(line: 10, scope: !14) diff --git a/llvm/test/DebugInfo/PDB/Inputs/typedef.yaml b/llvm/test/DebugInfo/PDB/Inputs/typedef.yaml new file mode 100644 index 000000000000..c02b11cce613 --- /dev/null +++ b/llvm/test/DebugInfo/PDB/Inputs/typedef.yaml @@ -0,0 +1,12 @@ +--- +TpiStream: + Records: + - Kind: LF_ALIAS + Alias: + UnderlyingType: 32 + Name: u8 + - Kind: LF_ALIAS + Alias: + UnderlyingType: 19 + Name: i64 +... diff --git a/llvm/test/DebugInfo/PDB/typedef.test b/llvm/test/DebugInfo/PDB/typedef.test new file mode 100644 index 000000000000..666036f27b03 --- /dev/null +++ b/llvm/test/DebugInfo/PDB/typedef.test @@ -0,0 +1,62 @@ +Built with clang (22+) because MSVC does not output lf_alias for typedefs + +Build typedef.ll with clang -S -emit-llvm -g: + +``` +void *__purecall = 0; + +typedef unsigned char u8; +using i64 = long long; + +int main() { + u8 val = 15; + i64 val2 = -1; + + return 0; +} +``` + +RUN: llc -O0 %p/Inputs/typedef.ll --filetype=obj -o %t.obj +RUN: llvm-pdbutil dump --types %t.obj | FileCheck --check-prefix=OBJTYPE %s + +OBJTYPE: 0x1003 | LF_ALIAS [size = 12] alias = u8, underlying type = 0x0020 (unsigned char) +OBJTYPE-NEXT: 0x1004 | LF_ALIAS [size = 12] alias = i64, underlying type = 0x0013 (__int64) + +RUN: llvm-pdbutil dump --symbols %t.obj | FileCheck --check-prefix=OBJSYM %s + +OBJSYM: 0 | S_LOCAL [size = 16] `val` +OBJSYM-NEXT: type=0x1003 (u8), flags = none +OBJSYM: 0 | S_LOCAL [size = 16] `val2` +OBJSYM: type=0x1004 (i64), flags = none +OBJSYM: 0 | S_UDT [size = 12] `u8` +OBJSYM-NEXT: original type = 0x1003 +OBJSYM-NEXT: 0 | S_UDT [size = 12] `i64` +OBJSYM-NEXT: original type = 0x1004 + + +typedef.yaml file generated via `clang -fno-rtti -g -O0 typedef.cpp` and `llvm-pdbutil pdb2yaml -tpi-stream`, and then manually +reduced to only the typedef nodes + +RUN: llvm-pdbutil yaml2pdb -pdb=%t.yaml.pdb %p/Inputs/typedef.yaml +RUN: llvm-pdbutil dump --types %t.yaml.pdb \ +RUN: | FileCheck --check-prefix=TYPES %s + +Also check that the YAML output hasn't drifted +RUN: llvm-pdbutil pdb2yaml -tpi-stream %t.yaml.pdb \ +RUN: | FileCheck --check-prefixes=YAML %s + +TYPES: Types (TPI Stream) +TYPES-NEXT:============================================================ +TYPES-NEXT: Showing 2 records +TYPES-NEXT: 0x1000 | LF_ALIAS [size = 12] alias = u8, underlying type = 0x0020 (unsigned char) +TYPES-NEXT: 0x1001 | LF_ALIAS [size = 12] alias = i64, underlying type = 0x0013 (__int64) + + +YAML: - Kind: LF_ALIAS +YAML-NEXT: Alias: +YAML-NEXT: UnderlyingType: 32 +YAML-NEXT: Name: u8 +YAML-NEXT: - Kind: LF_ALIAS +YAML-NEXT: Alias: +YAML-NEXT: UnderlyingType: 19 +YAML-NEXT: Name: i64 diff --git a/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll b/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll new file mode 100644 index 000000000000..b8259dcfab85 --- /dev/null +++ b/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll @@ -0,0 +1,48 @@ +; RUN: opt -passes=instcombine -S %s | FileCheck %s + +;; Instcombine folds the trunc %x into the phi. Check it updates the phi's dbg +;; user (otherwise the dbg use becomes poison after the original phi is +;; deleted). Check the new phi inherits the DebugLoc. + +; CHECK: %[[phi:.*]] = phi i8 [ 1, %{{.*}} ], [ 0, %{{.*}} ], !dbg ![[dbg:[0-9]+]] +; CHECK: call void @llvm.dbg.value(metadata i8 %[[phi]], metadata ![[#]], metadata !DIExpression(DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed, DW_OP_stack_value) +; CHECK: ![[dbg]] = !DILocation(line: 123, + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define linkonce_odr float @f(i1 %cond) { +entry: + br i1 %cond, label %if.then, label %if.end + +if.then: ; preds = entry + br label %if.end + +if.end: ; preds = %if.then, %entry + %p.0 = phi i32 [ 1, %if.then ], [ 0, %entry ], !dbg !13 + call void @llvm.dbg.value(metadata i32 %p.0, metadata !4, metadata !DIExpression()), !dbg !13 + %x = trunc i32 %p.0 to i8 + %callff = call float @ff(i8 %x) + ret float %callff +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) +declare float @ff(float) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_11, file: !1, producer: "clang version 17.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "btConvexTriangleMeshShape.cpp", directory: "/") +!2 = !{} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !DILocalVariable(name: "p", scope: !5, file: !6, line: 2057, type: !12) +!5 = distinct !DILexicalBlock(scope: !7, file: !6, line: 2056, column: 4) +!6 = !DIFile(filename: "reduce.cpp", directory: "/") +!7 = distinct !DISubprogram(name: "diagonalize", linkageName: "_ZN11btMatrix3x311diagonalizeERS_fi", scope: !8, file: !6, line: 2054, type: !9, scopeLine: 2055, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !11, retainedNodes: !2) +!8 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "btMatrix3x3", file: !6, line: 2050, size: 24, flags: DIFlagTypePassByValue, elements: !2, identifier: "_ZTS11btMatrix3x3") +!9 = !DISubroutineType(types: !10) +!10 = !{null} +!11 = !DISubprogram(name: "diagonalize", linkageName: "_ZN11btMatrix3x311diagonalizeERS_fi", scope: !8, file: !6, line: 2054, type: !9, scopeLine: 2054, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagOptimized) +!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!13 = !DILocation(line: 123, scope: !5) diff --git a/llvm/test/MC/AArch64/basic-a64-instructions.s b/llvm/test/MC/AArch64/basic-a64-instructions.s index 89f5944e02c5..f9bba53f2b83 100644 --- a/llvm/test/MC/AArch64/basic-a64-instructions.s +++ b/llvm/test/MC/AArch64/basic-a64-instructions.s @@ -3443,12 +3443,8 @@ _func: adr x9, #1048575 adr x2, #-1048576 - adrp x9, #4294963200 - adrp x20, #-4294967296 // CHECK: adr x9, #1048575 // encoding: [0xe9,0xff,0x7f,0x70] // CHECK: adr x2, #-1048576 // encoding: [0x02,0x00,0x80,0x10] -// CHECK: adrp x9, #4294963200 // encoding: [0xe9,0xff,0x7f,0xf0] -// CHECK: adrp x20, #-4294967296 // encoding: [0x14,0x00,0x80,0x90] //------------------------------------------------------------------------------ // System diff --git a/llvm/test/MC/Disassembler/AArch64/marked-up.txt b/llvm/test/MC/Disassembler/AArch64/marked-up.txt index 212187d2c3d5..137720d68bef 100644 --- a/llvm/test/MC/Disassembler/AArch64/marked-up.txt +++ b/llvm/test/MC/Disassembler/AArch64/marked-up.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc -triple=aarch64 -mattr=+all --mdis %s | FileCheck %s -# CHECK: b.ne +# CHECK: b.ne 0xa1 0x00 0x00 0x54 # CHECK-NEXT: asr , , 0x00 0x7c 0x00 0x13 diff --git a/llvm/test/MC/Disassembler/ARM/marked-up-thumb.txt b/llvm/test/MC/Disassembler/ARM/marked-up-thumb.txt index 65be28618bac..67f05bb1ea5b 100644 --- a/llvm/test/MC/Disassembler/ARM/marked-up-thumb.txt +++ b/llvm/test/MC/Disassembler/ARM/marked-up-thumb.txt @@ -1,5 +1,5 @@ # RUN: llvm-mc -triple=thumbv7-apple-darwin -mcpu=cortex-a8 -mdis < %s | FileCheck %s -# CHECK: ldr , ]> +# CHECK: ldr , , ]> 0x08 0x4c # CHECK: push {, , } 0x86 0xb4 diff --git a/llvm/test/MC/Disassembler/Mips/marked-up.txt b/llvm/test/MC/Disassembler/Mips/marked-up.txt index ef805c0a1cde..94935276af7e 100644 --- a/llvm/test/MC/Disassembler/Mips/marked-up.txt +++ b/llvm/test/MC/Disassembler/Mips/marked-up.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc --mdis %s -triple=mips-unknown-linux 2>&1 | FileCheck %s -# CHECK: j +# CHECK: j 0x09 0x33 0x00 0x2a # CHECK: addi , , 0x21 0x2d 0x66 0xd2 diff --git a/llvm/test/MC/Disassembler/SystemZ/marked-up.txt b/llvm/test/MC/Disassembler/SystemZ/marked-up.txt index c75f4e60c829..f3413e3091a5 100644 --- a/llvm/test/MC/Disassembler/SystemZ/marked-up.txt +++ b/llvm/test/MC/Disassembler/SystemZ/marked-up.txt @@ -2,7 +2,7 @@ # CHECK: blr 0x07 0x4a -# CHECK: lrl , +# CHECK: lrl , 0xc4 0xfd 0x00 0x00 0x00 0x47 # CHECK: stc , 0x42 0x00 0x0f 0xff diff --git a/llvm/test/MC/Disassembler/X86/marked-up-32.txt b/llvm/test/MC/Disassembler/X86/marked-up-32.txt index 20fe31538848..363c503a029b 100644 --- a/llvm/test/MC/Disassembler/X86/marked-up-32.txt +++ b/llvm/test/MC/Disassembler/X86/marked-up-32.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc -triple=i386 --mdis %s | FileCheck %s -# CHECK: je +# CHECK: je 0x0f 0x84 0x3c 0x00 0x00 0x00 -# CHECK: jecxz +# CHECK: jecxz 0xe3 0x81 diff --git a/llvm/test/MC/Disassembler/X86/marked-up.txt b/llvm/test/MC/Disassembler/X86/marked-up.txt index f0e51252f8d8..7b1a4952ea87 100644 --- a/llvm/test/MC/Disassembler/X86/marked-up.txt +++ b/llvm/test/MC/Disassembler/X86/marked-up.txt @@ -1,6 +1,6 @@ # RUN: llvm-mc --mdis %s -triple=x86_64-apple-darwin9 2>&1 | FileCheck %s -# CHECK: movq :8>, +# CHECK: movq :>, 0x65 0x48 0x8b 0x0c 0x25 0x08 0x00 0x00 0x00 # CHECK: xorps , 0x0f 0x57 0xd1 diff --git a/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml b/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml index 886597a0129b..092e5c1e9c57 100644 --- a/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml +++ b/llvm/test/ObjectYAML/MachO/DWARF-BigEndian.yaml @@ -272,7 +272,7 @@ DWARF: - long long int - __absvdi2 - a - - N + - 'N' - t debug_abbrev: - Table: @@ -384,7 +384,7 @@ DWARF: #CHECK: - long long int #CHECK: - __absvdi2 #CHECK: - a -#CHECK: - N +#CHECK: - 'N' #CHECK: - t #CHECK: debug_abbrev: #CHECK: - Code: 0x1 diff --git a/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml b/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml index 12bf9e70c982..04be63178a24 100644 --- a/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml +++ b/llvm/test/ObjectYAML/MachO/DWARF-LittleEndian.yaml @@ -261,7 +261,7 @@ DWARF: - long long int - __absvdi2 - a - - N + - 'N' - t debug_abbrev: - Table: @@ -373,7 +373,7 @@ DWARF: #CHECK: - long long int #CHECK: - __absvdi2 #CHECK: - a -#CHECK: - N +#CHECK: - 'N' #CHECK: - t #CHECK: debug_abbrev: #CHECK: - Code: 0x1 diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll index 435a096a6e1e..0d3b547f102b 100644 --- a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll +++ b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll @@ -946,5 +946,70 @@ define i1 @intrinsic_range(i16 %x) { ret i1 %res } +define i1 @supported_intrinsic_range(i16 %x) { +; CHECK-LABEL: @supported_intrinsic_range( +; CHECK-NEXT: [[ABS:%.*]] = call i16 @llvm.abs.i16(i16 [[X:%.*]], i1 false), !range [[RNG5]] +; CHECK-NEXT: [[TRUNC:%.*]] = trunc i16 [[ABS]] to i8 +; CHECK-NEXT: ret i1 true +; + %abs = call i16 @llvm.abs.i16(i16 %x, i1 false), !range !{i16 0, i16 8} + %trunc = trunc i16 %abs to i8 + %res = icmp ult i8 %trunc, 8 + ret i1 %res +} + +define i1 @ctlz_fold(i16 %x) { +; CHECK-LABEL: @ctlz_fold( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i16 [[X:%.*]], 256 +; CHECK-NEXT: br i1 [[CMP]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK: if: +; CHECK-NEXT: [[CTLZ:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: ret i1 true +; CHECK: else: +; CHECK-NEXT: [[CTLZ2:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: ret i1 true +; + %cmp = icmp ult i16 %x, 256 + br i1 %cmp, label %if, label %else + +if: + %ctlz = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res = icmp uge i16 %ctlz, 8 + ret i1 %res + +else: + %ctlz2 = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res2 = icmp ult i16 %ctlz2, 8 + ret i1 %res2 +} + +define i1 @ctlz_nofold(i16 %x) { +; CHECK-LABEL: @ctlz_nofold( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i16 [[X:%.*]], 256 +; CHECK-NEXT: br i1 [[CMP]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK: if: +; CHECK-NEXT: [[CTLZ:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: [[RES:%.*]] = icmp uge i16 [[CTLZ]], 9 +; CHECK-NEXT: ret i1 [[RES]] +; CHECK: else: +; CHECK-NEXT: [[CTLZ2:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X]], i1 false) +; CHECK-NEXT: [[RES2:%.*]] = icmp ult i16 [[CTLZ2]], 7 +; CHECK-NEXT: ret i1 [[RES2]] +; + %cmp = icmp ult i16 %x, 256 + br i1 %cmp, label %if, label %else + +if: + %ctlz = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res = icmp uge i16 %ctlz, 9 + ret i1 %res + +else: + %ctlz2 = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + %res2 = icmp ult i16 %ctlz2, 7 + ret i1 %res2 +} + declare i16 @llvm.ctlz.i16(i16, i1) +declare i16 @llvm.abs.i16(i16, i1) declare void @llvm.assume(i1) diff --git a/llvm/test/Transforms/SimplifyCFG/preserving-debugloc-br.ll b/llvm/test/Transforms/SimplifyCFG/preserving-debugloc-br.ll new file mode 100644 index 000000000000..298701ff61b4 --- /dev/null +++ b/llvm/test/Transforms/SimplifyCFG/preserving-debugloc-br.ll @@ -0,0 +1,72 @@ +; RUN: opt -S -passes=simplifycfg < %s | FileCheck %s + +; Check that SimplifyCFGPass's performBlockTailMerging() propagates +; the debug location of the old block terminator to the new branch +; instruction. + +define i32 @foo(i64 %x, i64 %y) !dbg !5 { +; CHECK-LABEL: define i32 @foo( +; CHECK: a: +; CHECK: br label %[[COMMON_RET:.*]], !dbg [[DBG14:![0-9]+]] +; CHECK: b: +; CHECK: br label %[[COMMON_RET]], !dbg [[DBG17:![0-9]+]] +; +entry: + %eq = icmp eq i64 %x, %y, !dbg !8 + br i1 %eq, label %b, label %switch, !dbg !9 + +switch: ; preds = %entry + %lt = icmp slt i64 %x, %y, !dbg !10 + %qux = select i1 %lt, i32 0, i32 2, !dbg !11 + switch i32 %qux, label %bees [ + i32 0, label %a + i32 1, label %b + i32 2, label %b + ], !dbg !12 + +a: ; preds = %switch + tail call void @bees.a(), !dbg !13 + ret i32 1, !dbg !14 + +b: ; preds = %switch, %switch, %entry + %retval = phi i32 [ 0, %switch ], [ 0, %switch ], [ 2, %entry ], !dbg !15 + tail call void @bees.b(), !dbg !16 + ret i32 %retval, !dbg !17 + +bees: ; preds = %switch + tail call void @llvm.trap(), !dbg !18 + unreachable, !dbg !19 +} + + +declare void @llvm.trap() +declare void @bees.a() +declare void @bees.b() + +!llvm.dbg.cu = !{!0} +!llvm.debugify = !{!2, !3} +!llvm.module.flags = !{!4} + +; CHECK: [[DBG14]] = !DILocation(line: 7, +; CHECK: [[DBG17]] = !DILocation(line: 10, + +!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) +!1 = !DIFile(filename: "main.ll", directory: "/") +!2 = !{i32 12} +!3 = !{i32 0} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0) +!6 = !DISubroutineType(types: !7) +!7 = !{} +!8 = !DILocation(line: 1, column: 1, scope: !5) +!9 = !DILocation(line: 2, column: 1, scope: !5) +!10 = !DILocation(line: 3, column: 1, scope: !5) +!11 = !DILocation(line: 4, column: 1, scope: !5) +!12 = !DILocation(line: 5, column: 1, scope: !5) +!13 = !DILocation(line: 6, column: 1, scope: !5) +!14 = !DILocation(line: 7, column: 1, scope: !5) +!15 = !DILocation(line: 8, column: 1, scope: !5) +!16 = !DILocation(line: 9, column: 1, scope: !5) +!17 = !DILocation(line: 10, column: 1, scope: !5) +!18 = !DILocation(line: 11, column: 1, scope: !5) +!19 = !DILocation(line: 12, column: 1, scope: !5) diff --git a/llvm/test/YAMLParser/bool.test b/llvm/test/YAMLParser/bool.test index e987a0ec1e32..885c78c6660f 100644 --- a/llvm/test/YAMLParser/bool.test +++ b/llvm/test/YAMLParser/bool.test @@ -1,6 +1,24 @@ # RUN: yaml-bench -canonical %s -- yes -- NO +- true - True +- TRUE +- yes +- Yes +- YES - on +- On +- ON +- false +- False +- FALSE +- no +- No +- NO +- off +- Off +- OFF +- y +- Y +- n +- N diff --git a/llvm/test/tools/llvm-objdump/MachO/Inputs/macho-rebase-set-type-imm b/llvm/test/tools/llvm-objdump/MachO/Inputs/macho-rebase-set-type-imm deleted file mode 100755 index 947db0ee915f..000000000000 Binary files a/llvm/test/tools/llvm-objdump/MachO/Inputs/macho-rebase-set-type-imm and /dev/null differ diff --git a/llvm/test/tools/llvm-objdump/MachO/bad-bind.test b/llvm/test/tools/llvm-objdump/MachO/bad-bind.test index d11c4fb88395..f8d1ac6d8d22 100644 --- a/llvm/test/tools/llvm-objdump/MachO/bad-bind.test +++ b/llvm/test/tools/llvm-objdump/MachO/bad-bind.test @@ -67,9 +67,6 @@ WEAK-BIND-SET-DYLIB-ORDINAL-ULEB: macho-weak-bind-set-dylib-ordinal-uleb': trunc RUN: not llvm-objdump --macho --weak-bind %p/Inputs/macho-weak-bind-set-dylib-special-imm 2>&1 | FileCheck --check-prefix WEAK-BIND-SET-DYLIB-SPECIAL-IMM %s WEAK-BIND-SET-DYLIB-SPECIAL-IMM: macho-weak-bind-set-dylib-special-imm': truncated or malformed object (BIND_OPCODE_SET_DYLIB_SPECIAL_IMM not allowed in weak bind table for opcode at: 0x2) -RUN: not llvm-objdump --macho --rebase %p/Inputs/macho-rebase-set-type-imm 2>&1 | FileCheck --check-prefix REBASE-SET-TYPE-IMM %s -REBASE-SET-TYPE-IMM: macho-rebase-set-type-imm': truncated or malformed object (for REBASE_OPCODE_SET_TYPE_IMM bad bind type: 5 for opcode at: 0x0) - RUN: not llvm-objdump --macho --rebase %p/Inputs/macho-rebase-uleb-malformed-uleb128 2>&1 | FileCheck --check-prefix REBASE-ULEB-MALFORMED-ULEB128 %s REBASE-ULEB-MALFORMED-ULEB128: macho-rebase-uleb-malformed-uleb128': truncated or malformed object (for REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB malformed uleb128, extends past end for opcode at: 0x1) diff --git a/llvm/test/tools/llvm-size/common.test b/llvm/test/tools/llvm-size/common.test index ac804778ed76..becc183875b5 100644 --- a/llvm/test/tools/llvm-size/common.test +++ b/llvm/test/tools/llvm-size/common.test @@ -34,7 +34,7 @@ Sections: - Name: .text Type: SHT_PROGBITS Symbols: - - Name: y + - Name: 'y' Type: STT_OBJECT Size: 4 Index: SHN_COMMON diff --git a/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp b/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp index be7e487673fb..e33628ea8340 100644 --- a/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp +++ b/llvm/tools/llvm-pdbutil/MinimalTypeDumper.cpp @@ -521,6 +521,13 @@ Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, return Error::success(); } +Error MinimalTypeDumpVisitor::visitKnownRecord(CVType &CVR, + AliasRecord &Alias) { + P.format(" alias = {0}, underlying type = {1}", Alias.Name, + Alias.UnderlyingType); + return Error::success(); +} + Error MinimalTypeDumpVisitor::visitKnownMember(CVMemberRecord &CVR, NestedTypeRecord &Nested) { P.format(" [name = `{0}`, parent = {1}]", Nested.Name, Nested.Type); diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt index f20bc2eb7dcc..8f0a3c501b52 100644 --- a/llvm/unittests/ADT/CMakeLists.txt +++ b/llvm/unittests/ADT/CMakeLists.txt @@ -56,12 +56,14 @@ add_llvm_unittest(ADTTests PointerUnionTest.cpp PostOrderIteratorTest.cpp PriorityWorklistTest.cpp + RadixTreeTest.cpp RangeAdapterTest.cpp SCCIteratorTest.cpp STLExtrasTest.cpp STLForwardCompatTest.cpp ScopeExitTest.cpp SequenceTest.cpp + SetOperationsTest.cpp SetVectorTest.cpp SimpleIListTest.cpp SmallPtrSetTest.cpp diff --git a/llvm/unittests/ADT/IteratorTest.cpp b/llvm/unittests/ADT/IteratorTest.cpp index 641ff4fcc7f5..67504c7fbf5f 100644 --- a/llvm/unittests/ADT/IteratorTest.cpp +++ b/llvm/unittests/ADT/IteratorTest.cpp @@ -11,10 +11,12 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include using namespace llvm; +using testing::ElementsAre; namespace { @@ -391,6 +393,21 @@ TEST(PointerIterator, Range) { EXPECT_EQ(A + I++, P); } +namespace rbegin_detail { +struct WithFreeRBegin { + int data[3] = {42, 43, 44}; +}; + +auto rbegin(const WithFreeRBegin &X) { return std::rbegin(X.data); } +auto rend(const WithFreeRBegin &X) { return std::rend(X.data); } +} // namespace rbegin_detail + +TEST(ReverseTest, ADL) { + // Check that we can find the rbegin/rend functions via ADL. + rbegin_detail::WithFreeRBegin Foo; + EXPECT_THAT(reverse(Foo), ElementsAre(44, 43, 42)); +} + TEST(ZipIteratorTest, Basic) { using namespace std; const SmallVector pi{3, 1, 4, 1, 5, 9}; diff --git a/llvm/unittests/ADT/RadixTreeTest.cpp b/llvm/unittests/ADT/RadixTreeTest.cpp new file mode 100644 index 000000000000..54f4697c2f3e --- /dev/null +++ b/llvm/unittests/ADT/RadixTreeTest.cpp @@ -0,0 +1,385 @@ +//===- llvm/unittest/ADT/RadixTreeTest.cpp --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/RadixTree.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include +#include +#include + +using namespace llvm; +namespace { + +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +// Wrapper for ElementsAreArray that works with ranges lacking ::value_type +// (e.g., iterator_range). +template auto RangeElementsAre(const Range &R) { + return ElementsAreArray(adl_begin(R), adl_end(R)); +} + +// Test with StringRef. + +TEST(RadixTreeTest, Empty) { + RadixTree T; + EXPECT_TRUE(T.empty()); + EXPECT_EQ(T.size(), 0u); + + EXPECT_TRUE(T.find_prefixes("").empty()); + EXPECT_TRUE(T.find_prefixes("A").empty()); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TEST(RadixTreeTest, InsertEmpty) { + RadixTree T; + auto [It, IsNew] = T.emplace("", 4); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(IsNew); + const auto &[K, V] = *It; + EXPECT_TRUE(K.empty()); + EXPECT_EQ(4, V); + + EXPECT_THAT(T, ElementsAre(Pair("", 4))); + + EXPECT_THAT(T.find_prefixes(""), ElementsAre(Pair("", 4))); + + EXPECT_THAT(T.find_prefixes("a"), ElementsAre(Pair("", 4))); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TEST(RadixTreeTest, Complex) { + RadixTree T; + T.emplace("abcd", 1); + EXPECT_EQ(T.countNodes(), 2u); + T.emplace("abklm", 2); + EXPECT_EQ(T.countNodes(), 4u); + T.emplace("123abklm", 3); + EXPECT_EQ(T.countNodes(), 5u); + T.emplace("123abklm", 4); + EXPECT_EQ(T.countNodes(), 5u); + T.emplace("ab", 5); + EXPECT_EQ(T.countNodes(), 5u); + T.emplace("1234567", 6); + EXPECT_EQ(T.countNodes(), 7u); + T.emplace("123456", 7); + EXPECT_EQ(T.countNodes(), 8u); + T.emplace("123456789", 8); + EXPECT_EQ(T.countNodes(), 9u); + + EXPECT_THAT(T, UnorderedElementsAre(Pair("abcd", 1), Pair("abklm", 2), + Pair("123abklm", 3), Pair("ab", 5), + Pair("1234567", 6), Pair("123456", 7), + Pair("123456789", 8))); + + EXPECT_THAT(T.find_prefixes("1234567890"), + UnorderedElementsAre(Pair("1234567", 6), Pair("123456", 7), + Pair("123456789", 8))); + + EXPECT_THAT(T.find_prefixes("123abklm"), + UnorderedElementsAre(Pair("123abklm", 3))); + + EXPECT_THAT(T.find_prefixes("abcdefg"), + UnorderedElementsAre(Pair("abcd", 1), Pair("ab", 5))); + + EXPECT_EQ(T.countNodes(), 9u); +} + +TEST(RadixTreeTest, ValueWith2Parameters) { + RadixTree> T; + T.emplace("abcd", "a", 3); + + EXPECT_THAT(T, UnorderedElementsAre(Pair("abcd", Pair("a", 3)))); +} + +// Test different types, less readable. + +template struct TestData { + static const T Data1[]; + static const T Data2[]; +}; + +template <> const char TestData::Data1[] = "abcdedcba"; +template <> const char TestData::Data2[] = "abCDEDCba"; + +template <> const int TestData::Data1[] = {1, 2, 3, 4, 5, 4, 3, 2, 1}; +template <> const int TestData::Data2[] = {1, 2, 4, 8, 16, 8, 4, 2, 1}; + +template class RadixTreeTypeTest : public ::testing::Test { +public: + using IteratorType = decltype(adl_begin(std::declval())); + using CharType = remove_cvref_t()))>; + + T make(const CharType *Data, size_t N) { return T(StringRef(Data, N)); } + + T make1(size_t N) { return make(TestData::Data1, N); } + T make2(size_t N) { return make(TestData::Data2, N); } +}; + +template <> +iterator_range +RadixTreeTypeTest>::make( + const char *Data, size_t N) { + return StringRef(Data).take_front(N); +} + +template <> +iterator_range +RadixTreeTypeTest>::make( + const char *Data, size_t N) { + return reverse(StringRef(Data).take_back(N)); +} + +template <> +ArrayRef RadixTreeTypeTest>::make(const int *Data, + size_t N) { + return ArrayRef(Data, Data + N); +} + +template <> +std::vector RadixTreeTypeTest>::make(const int *Data, + size_t N) { + return std::vector(Data, Data + N); +} + +template <> +std::list RadixTreeTypeTest>::make(const int *Data, + size_t N) { + return std::list(Data, Data + N); +} + +class TypeNameGenerator { +public: + template static std::string GetName(int) { + if (std::is_same_v) + return "StringRef"; + if (std::is_same_v) + return "string"; + if (std::is_same_v>) + return "iterator_range"; + if (std::is_same_v>) + return "reverse_iterator_range"; + if (std::is_same_v>) + return "ArrayRef"; + if (std::is_same_v>) + return "vector"; + if (std::is_same_v>) + return "list"; + return "Unknown"; + } +}; + +using TestTypes = + ::testing::Types, + iterator_range, + ArrayRef, std::vector, std::list>; + +TYPED_TEST_SUITE(RadixTreeTypeTest, TestTypes, TypeNameGenerator); + +TYPED_TEST(RadixTreeTypeTest, Helpers) { + for (size_t i = 0; i < 9; ++i) { + auto R1 = this->make1(i); + auto R2 = this->make2(i); + EXPECT_EQ(llvm::range_size(R1), i); + EXPECT_EQ(llvm::range_size(R2), i); + auto [I1, I2] = llvm::mismatch(R1, R2); + // Exactly 2 first elements of Data1 and Data2 must match. + EXPECT_EQ(std::distance(R1.begin(), I1), std::min(2, i)); + } +} + +TYPED_TEST(RadixTreeTypeTest, Empty) { + RadixTree T; + EXPECT_TRUE(T.empty()); + EXPECT_EQ(T.size(), 0u); + + EXPECT_TRUE(T.find_prefixes(this->make1(0)).empty()); + EXPECT_TRUE(T.find_prefixes(this->make2(1)).empty()); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertEmpty) { + using TreeType = RadixTree; + TreeType T; + auto [It, IsNew] = T.emplace(this->make1(0), 5); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(IsNew); + const auto &[K, V] = *It; + EXPECT_TRUE(K.empty()); + EXPECT_EQ(5, V); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T.find_prefixes(this->make2(1)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T, ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertEmptyTwice) { + using TreeType = RadixTree; + TreeType T; + T.emplace(this->make1(0), 5); + auto [It, IsNew] = T.emplace(this->make1(0), 6); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(!IsNew); + const auto &[K, V] = *It; + EXPECT_TRUE(K.empty()); + EXPECT_EQ(5, V); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T.find_prefixes(this->make2(1)), + ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_THAT(T, ElementsAre(Pair(ElementsAre(), 5))); + + EXPECT_EQ(T.countNodes(), 1u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertOne) { + using TreeType = RadixTree; + TreeType T; + auto [It, IsNew] = T.emplace(this->make1(1), 4); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(IsNew); + const auto &[K, V] = *It; + EXPECT_THAT(K, RangeElementsAre(this->make1(1))); + EXPECT_EQ(4, V); + + EXPECT_THAT(T, ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + + EXPECT_THAT(T.find_prefixes(this->make1(1)), + ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + + EXPECT_THAT(T.find_prefixes(this->make1(2)), + ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + + EXPECT_EQ(T.countNodes(), 2u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertOneTwice) { + using TreeType = RadixTree; + TreeType T; + T.emplace(this->make1(1), 4); + auto [It, IsNew] = T.emplace(this->make1(1), 4); + EXPECT_TRUE(!T.empty()); + EXPECT_EQ(T.size(), 1u); + EXPECT_TRUE(!IsNew); + + EXPECT_THAT(T, ElementsAre(Pair(RangeElementsAre(this->make1(1)), 4))); + EXPECT_EQ(T.countNodes(), 2u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertSuperStrings) { + using TreeType = RadixTree; + TreeType T; + + for (size_t Len = 0; Len < 7; Len += 2) { + auto [It, IsNew] = T.emplace(this->make1(Len), Len); + EXPECT_TRUE(IsNew); + } + + EXPECT_THAT(T, + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0), + Pair(RangeElementsAre(this->make1(2)), 2), + Pair(RangeElementsAre(this->make1(4)), 4), + Pair(RangeElementsAre(this->make1(6)), 6))); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0))); + + EXPECT_THAT(T.find_prefixes(this->make1(3)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0), + Pair(RangeElementsAre(this->make1(2)), 2))); + + EXPECT_THAT(T.find_prefixes(this->make1(7)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(0)), 0), + Pair(RangeElementsAre(this->make1(2)), 2), + Pair(RangeElementsAre(this->make1(4)), 4), + Pair(RangeElementsAre(this->make1(6)), 6))); + + EXPECT_EQ(T.countNodes(), 4u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertSubStrings) { + using TreeType = RadixTree; + TreeType T; + + for (size_t Len = 0; Len < 7; Len += 2) { + auto [It, IsNew] = T.emplace(this->make1(7 - Len), 7 - Len); + EXPECT_TRUE(IsNew); + } + + EXPECT_THAT(T, + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(1)), 1), + Pair(RangeElementsAre(this->make1(3)), 3), + Pair(RangeElementsAre(this->make1(5)), 5), + Pair(RangeElementsAre(this->make1(7)), 7))); + + EXPECT_THAT(T.find_prefixes(this->make1(0)), UnorderedElementsAre()); + + EXPECT_THAT(T.find_prefixes(this->make1(3)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(1)), 1), + Pair(RangeElementsAre(this->make1(3)), 3))); + + EXPECT_THAT(T.find_prefixes(this->make1(6)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(1)), 1), + Pair(RangeElementsAre(this->make1(3)), 3), + Pair(RangeElementsAre(this->make1(5)), 5))); + + EXPECT_EQ(T.countNodes(), 5u); +} + +TYPED_TEST(RadixTreeTypeTest, InsertVShape) { + using TreeType = RadixTree; + TreeType T; + + EXPECT_EQ(T.countNodes(), 1u); + T.emplace(this->make1(5), 15); + EXPECT_EQ(T.countNodes(), 2u); + T.emplace(this->make2(6), 26); + EXPECT_EQ(T.countNodes(), 4u); + T.emplace(this->make2(1), 21); + EXPECT_EQ(T.countNodes(), 5u); + + EXPECT_THAT(T, + UnorderedElementsAre(Pair(RangeElementsAre(this->make1(5)), 15), + Pair(RangeElementsAre(this->make2(6)), 26), + Pair(RangeElementsAre(this->make2(1)), 21))); + + EXPECT_THAT(T.find_prefixes(this->make1(7)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make2(1)), 21), + Pair(RangeElementsAre(this->make1(5)), 15))); + + EXPECT_THAT(T.find_prefixes(this->make2(7)), + UnorderedElementsAre(Pair(RangeElementsAre(this->make2(1)), 21), + Pair(RangeElementsAre(this->make2(6)), 26))); + + EXPECT_EQ(T.countNodes(), 5u); +} + +} // namespace diff --git a/llvm/unittests/ADT/RangeAdapterTest.cpp b/llvm/unittests/ADT/RangeAdapterTest.cpp index 3c68103e1745..5b8a99dd42c8 100644 --- a/llvm/unittests/ADT/RangeAdapterTest.cpp +++ b/llvm/unittests/ADT/RangeAdapterTest.cpp @@ -151,10 +151,6 @@ TYPED_TEST(RangeAdapterRValueTest, TrivialOperation) { TestRev(reverse(TypeParam({0, 1, 2, 3}))); } -TYPED_TEST(RangeAdapterRValueTest, HasRbegin) { - static_assert(has_rbegin::value, "rbegin() should be defined"); -} - TYPED_TEST(RangeAdapterRValueTest, RangeType) { static_assert( std::is_same< diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp index 4a9c745f5328..ad2c9bdb0c8a 100644 --- a/llvm/unittests/ADT/STLExtrasTest.cpp +++ b/llvm/unittests/ADT/STLExtrasTest.cpp @@ -375,6 +375,14 @@ std::vector::const_iterator end(const some_struct &s) { return s.data.end(); } +std::vector::const_reverse_iterator rbegin(const some_struct &s) { + return s.data.rbegin(); +} + +std::vector::const_reverse_iterator rend(const some_struct &s) { + return s.data.rend(); +} + void swap(some_struct &lhs, some_struct &rhs) { // make swap visible as non-adl swap would even seem to // work with std::swap which defaults to moving @@ -389,6 +397,8 @@ TEST(STLExtrasTest, ADLTest) { EXPECT_EQ(*adl_begin(s), 1); EXPECT_EQ(*(adl_end(s) - 1), 5); + EXPECT_EQ(*adl_rbegin(s), 5); + EXPECT_EQ(*(adl_rend(s) - 1), 1); adl_swap(s, s2); EXPECT_EQ(s.swap_val, "lhs"); diff --git a/llvm/unittests/ADT/SetOperationsTest.cpp b/llvm/unittests/ADT/SetOperationsTest.cpp new file mode 100644 index 000000000000..982ea819fd80 --- /dev/null +++ b/llvm/unittests/ADT/SetOperationsTest.cpp @@ -0,0 +1,266 @@ +//===- SetOperations.cpp - Unit tests for set operations ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SetOperations.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include + +using namespace llvm; + +using testing::IsEmpty; + +namespace { + +TEST(SetOperationsTest, SetUnion) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {5, 6, 7, 8}; + // Set1 should be the union of input sets Set1 and Set2. + std::set ExpectedSet1 = {1, 2, 3, 4, 5, 6, 7, 8}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + + set_union(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1.clear(); + Set2 = {1, 2}; + // Set1 should be the union of input sets Set1 and Set2, which in this case + // will be Set2. + ExpectedSet1 = Set2; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_union(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetIntersect) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + // Set1 should be the intersection of sets Set1 and Set2. + std::set ExpectedSet1 = {3, 4}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + + set_intersect(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_intersect(Set1, Set2); + // Set1 should be the intersection of sets Set1 and Set2, which + // is empty as they are non-overlapping. + EXPECT_THAT(Set1, IsEmpty()); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetIntersection) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + std::set Result; + // Result should be the intersection of sets Set1 and Set2. + std::set ExpectedResult = {3, 4}; + // Set1 and Set2 should not be touched. + std::set ExpectedSet1 = Set1; + std::set ExpectedSet2 = Set2; + + Result = set_intersection(Set1, Set2); + EXPECT_EQ(ExpectedResult, Result); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_intersection(Set1, Set2); + // Result should be the intersection of sets Set1 and Set2, which + // is empty as they are non-overlapping. + EXPECT_THAT(Result, IsEmpty()); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {5, 6}; + Set2 = {1, 2, 3, 4}; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_intersection(Set1, Set2); + // Result should be the intersection of sets Set1 and Set2, which + // is empty as they are non-overlapping. Test this again with the input sets + // reversed, since the code takes a different path depending on which input + // set is smaller. + EXPECT_THAT(Result, IsEmpty()); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetDifference) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + std::set Result; + // Result should be Set1 - Set2, leaving only {1, 2}. + std::set ExpectedResult = {1, 2}; + // Set1 and Set2 should not be touched. + std::set ExpectedSet1 = Set1; + std::set ExpectedSet2 = Set2; + + Result = set_difference(Set1, Set2); + EXPECT_EQ(ExpectedResult, Result); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_difference(Set1, Set2); + // Result should be Set1 - Set2, which should be empty. + EXPECT_THAT(Result, IsEmpty()); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Result should be Set1 - Set2, which should be Set1 as they are + // non-overlapping. + ExpectedResult = Set1; + // Set1 and Set2 should not be touched. + ExpectedSet1 = Set1; + ExpectedSet2 = Set2; + + Result = set_difference(Set1, Set2); + EXPECT_EQ(ExpectedResult, Result); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetSubtract) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + // Set1 should get Set1 - Set2, leaving only {1, 2}. + std::set ExpectedSet1 = {1, 2}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + + set_subtract(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_subtract(Set1, Set2); + // Set1 should get Set1 - Set2, which should be empty. + EXPECT_THAT(Set1, IsEmpty()); + EXPECT_EQ(ExpectedSet2, Set2); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + // Set1 should get Set1 - Set2, which should be Set1 as they are + // non-overlapping. + ExpectedSet1 = Set1; + // Set2 should not be touched. + ExpectedSet2 = Set2; + + set_subtract(Set1, Set2); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); +} + +TEST(SetOperationsTest, SetSubtractRemovedRemaining) { + std::set Removed, Remaining; + + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4, 5, 6}; + // Set1 should get Set1 - Set2, leaving only {1, 2}. + std::set ExpectedSet1 = {1, 2}; + // Set2 should not be touched. + std::set ExpectedSet2 = Set2; + // We should get back that {3, 4} from Set2 were removed from Set1, and {5, 6} + // were not removed from Set1. + std::set ExpectedRemoved = {3, 4}; + std::set ExpectedRemaining = {5, 6}; + + set_subtract(Set1, Set2, Removed, Remaining); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + EXPECT_EQ(ExpectedRemoved, Removed); + EXPECT_EQ(ExpectedRemaining, Remaining); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + Removed.clear(); + Remaining.clear(); + // Set2 should not be touched. + ExpectedSet2 = Set2; + // Set should get back that all of Set2 was removed from Set1, and nothing + // left in Set2 was not removed from Set1. + ExpectedRemoved = Set2; + + set_subtract(Set1, Set2, Removed, Remaining); + // Set1 should get Set1 - Set2, which should be empty. + EXPECT_THAT(Set1, IsEmpty()); + EXPECT_EQ(ExpectedSet2, Set2); + EXPECT_EQ(ExpectedRemoved, Removed); + EXPECT_THAT(Remaining, IsEmpty()); + + Set1 = {1, 2, 3, 4}; + Set2 = {5, 6}; + Removed.clear(); + Remaining.clear(); + // Set1 should get Set1 - Set2, which should be Set1 as they are + // non-overlapping. + ExpectedSet1 = {1, 2, 3, 4}; + // Set2 should not be touched. + ExpectedSet2 = Set2; + // Set should get back that none of Set2 was removed from Set1, and all + // of Set2 was not removed from Set1. + ExpectedRemaining = Set2; + + set_subtract(Set1, Set2, Removed, Remaining); + EXPECT_EQ(ExpectedSet1, Set1); + EXPECT_EQ(ExpectedSet2, Set2); + EXPECT_THAT(Removed, IsEmpty()); + EXPECT_EQ(ExpectedRemaining, Remaining); +} + +TEST(SetOperationsTest, SetIsSubset) { + std::set Set1 = {1, 2, 3, 4}; + std::set Set2 = {3, 4}; + EXPECT_FALSE(set_is_subset(Set1, Set2)); + + Set1 = {1, 2, 3, 4}; + Set2 = {1, 2, 3, 4}; + EXPECT_TRUE(set_is_subset(Set1, Set2)); + + Set1 = {1, 2}; + Set2 = {1, 2, 3, 4}; + EXPECT_TRUE(set_is_subset(Set1, Set2)); + + Set1 = {1, 2}; + Set2 = {3, 4}; + EXPECT_FALSE(set_is_subset(Set1, Set2)); +} + +} // namespace diff --git a/llvm/unittests/ADT/StringRefTest.cpp b/llvm/unittests/ADT/StringRefTest.cpp index cad1974dd126..9e6eee368c0c 100644 --- a/llvm/unittests/ADT/StringRefTest.cpp +++ b/llvm/unittests/ADT/StringRefTest.cpp @@ -71,9 +71,17 @@ TEST(StringRefTest, EmptyInitializerList) { TEST(StringRefTest, Iteration) { StringRef S("hello"); - const char *p = "hello"; - for (const char *it = S.begin(), *ie = S.end(); it != ie; ++it, ++p) - EXPECT_EQ(*it, *p); + constexpr StringLiteral CS("hello"); + + // Note: Cannot use literal strings in equal() as iteration over a literal + // string includes the null terminator. + const std::string_view RefFwd("hello"); + const std::string_view RefRev("olleh"); + + EXPECT_TRUE(equal(S, RefFwd)); + EXPECT_TRUE(equal(CS, RefFwd)); + EXPECT_TRUE(equal(make_range(S.rbegin(), S.rend()), RefRev)); + EXPECT_TRUE(equal(make_range(CS.rbegin(), CS.rend()), RefRev)); } TEST(StringRefTest, StringOps) { diff --git a/llvm/unittests/IR/ConstantRangeTest.cpp b/llvm/unittests/IR/ConstantRangeTest.cpp index b54746b00536..5d1931ebf602 100644 --- a/llvm/unittests/IR/ConstantRangeTest.cpp +++ b/llvm/unittests/IR/ConstantRangeTest.cpp @@ -2397,6 +2397,21 @@ TEST_F(ConstantRangeTest, Abs) { }); } +TEST_F(ConstantRangeTest, Ctlz) { + TestUnaryOpExhaustive([](const ConstantRange &CR) { return CR.ctlz(); }, + [](const APInt &N) { + return APInt(N.getBitWidth(), N.countLeadingZeros()); + }); + + TestUnaryOpExhaustive( + [](const ConstantRange &CR) { return CR.ctlz(/*ZeroIsPoison=*/true); }, + [](const APInt &N) -> std::optional { + if (N.isZero()) + return std::nullopt; + return APInt(N.getBitWidth(), N.countLeadingZeros()); + }); +} + TEST_F(ConstantRangeTest, castOps) { ConstantRange A(APInt(16, 66), APInt(16, 128)); ConstantRange FpToI8 = A.castOp(Instruction::FPToSI, 8); diff --git a/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp b/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp index 27948c1ad6f1..60af64a7ea09 100644 --- a/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp +++ b/llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp @@ -25,8 +25,8 @@ using DomUpdate = DominatorTree::UpdateType; static_assert( std::is_same_v, "Trees differing only in IsPostDom should have the same update types"); -using DomSNCA = DomTreeBuilder::SemiNCAInfo; -using PostDomSNCA = DomTreeBuilder::SemiNCAInfo; +using DomSNCA = DomTreeBuilder::SemiNCAInfoOnView; +using PostDomSNCA = DomTreeBuilder::SemiNCAInfoOnView; const auto Insert = DominatorTree::Insert; const auto Delete = DominatorTree::Delete; diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt index c9b0d53bc3e9..ab690b16a4fb 100644 --- a/mlir/CMakeLists.txt +++ b/mlir/CMakeLists.txt @@ -22,7 +22,6 @@ endif() # Must go below project(..) include(GNUInstallDirs) -set(CMAKE_CXX_STANDARD 17) if(MLIR_STANDALONE_BUILD) find_package(LLVM CONFIG REQUIRED) diff --git a/mlir/docs/TargetLLVMIR.md b/mlir/docs/TargetLLVMIR.md index 5ba7adc2463a..03bf920a04a0 100644 --- a/mlir/docs/TargetLLVMIR.md +++ b/mlir/docs/TargetLLVMIR.md @@ -35,7 +35,7 @@ Conversion to the LLVM dialect from other dialects is the first step to produce LLVM IR. All non-trivial IR modifications are expected to happen at this stage or before. The conversion is *progressive*: most passes convert one dialect to the LLVM dialect and keep operations from other dialects intact. For example, -the `-convert-memref-to-llvm` pass will only convert operations from the +the `-finalize-memref-to-llvm` pass will only convert operations from the `memref` dialect but will not convert operations from other dialects even if they use or produce `memref`-typed values. diff --git a/mlir/docs/doxygen.cfg.in b/mlir/docs/doxygen.cfg.in index b91b242cffaa..5f817d8c5376 100644 --- a/mlir/docs/doxygen.cfg.in +++ b/mlir/docs/doxygen.cfg.in @@ -1154,7 +1154,7 @@ HTML_INDEX_NUM_ENTRIES = 100 # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_DOCSET = NO +GENERATE_DOCSET = YES # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider @@ -1170,7 +1170,7 @@ DOCSET_FEEDNAME = "Doxygen generated docs" # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_BUNDLE_ID = org.doxygen.Project +DOCSET_BUNDLE_ID = mlir # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style @@ -1178,7 +1178,7 @@ DOCSET_BUNDLE_ID = org.doxygen.Project # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. -DOCSET_PUBLISHER_ID = org.doxygen.Publisher +DOCSET_PUBLISHER_ID = Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. @@ -1253,7 +1253,7 @@ TOC_EXPAND = NO # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_QHP = @llvm_doxygen_generate_qhp@ +GENERATE_QHP = YES # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to @@ -1268,7 +1268,7 @@ QCH_FILE = @llvm_doxygen_qch_filename@ # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. -QHP_NAMESPACE = @llvm_doxygen_qhp_namespace@ +QHP_NAMESPACE = mlir # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual @@ -1338,7 +1338,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1469,7 +1469,7 @@ MATHJAX_CODEFILE = # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -SEARCHENGINE = @enable_searchengine@ +SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using Javascript. There @@ -2091,7 +2091,7 @@ CLASS_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -COLLABORATION_GRAPH = YES +COLLABORATION_GRAPH = NO # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. diff --git a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp index 9b5d1a737dbf..06e509639a91 100644 --- a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp +++ b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp @@ -206,7 +206,7 @@ void ToyToLLVMLoweringPass::runOnOperation() { populateAffineToStdConversionPatterns(patterns); populateSCFToControlFlowConversionPatterns(patterns); mlir::arith::populateArithToLLVMConversionPatterns(typeConverter, patterns); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); cf::populateControlFlowToLLVMConversionPatterns(typeConverter, patterns); populateFuncToLLVMConversionPatterns(typeConverter, patterns); diff --git a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp index 9b5d1a737dbf..06e509639a91 100644 --- a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp +++ b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp @@ -206,7 +206,7 @@ void ToyToLLVMLoweringPass::runOnOperation() { populateAffineToStdConversionPatterns(patterns); populateSCFToControlFlowConversionPatterns(patterns); mlir::arith::populateArithToLLVMConversionPatterns(typeConverter, patterns); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); cf::populateControlFlowToLLVMConversionPatterns(typeConverter, patterns); populateFuncToLLVMConversionPatterns(typeConverter, patterns); diff --git a/mlir/include/mlir/Bytecode/BytecodeImplementation.h b/mlir/include/mlir/Bytecode/BytecodeImplementation.h index 1ae839d5f40e..ed40be6a6f43 100644 --- a/mlir/include/mlir/Bytecode/BytecodeImplementation.h +++ b/mlir/include/mlir/Bytecode/BytecodeImplementation.h @@ -39,6 +39,8 @@ class DialectBytecodeReader { /// Emit an error to the reader. virtual InFlightDiagnostic emitError(const Twine &msg = {}) = 0; + virtual MLIRContext *getContext() const = 0; + /// Read out a list of elements, invoking the provided callback for each /// element. The callback function may be in any of the following forms: /// * LogicalResult(T &) diff --git a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h index f27f7bb5975e..95b12b69153a 100644 --- a/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h +++ b/mlir/include/mlir/Conversion/ArithCommon/AttrToLLVMConverter.h @@ -50,28 +50,6 @@ class AttrConvertFastMathToLLVM { ArrayRef getAttrs() const { return convertedAttr.getAttrs(); } -private: - NamedAttrList convertedAttr; -}; - -// Attribute converter that populates a NamedAttrList by removing the fastmath -// attribute from the source operation attributes. This may be useful for -// target operations that do not require the fastmath attribute, or for targets -// that do not yet support the LLVM fastmath attribute. -template -class AttrDropFastMath { -public: - AttrDropFastMath(SourceOp srcOp) { - // Copy the source attributes. - convertedAttr = NamedAttrList{srcOp->getAttrs()}; - // Get the name of the arith fastmath attribute. - llvm::StringRef arithFMFAttrName = SourceOp::getFastMathAttrName(); - // Remove the source fastmath attribute. - convertedAttr.erase(arithFMFAttrName); - } - - ArrayRef getAttrs() const { return convertedAttr.getAttrs(); } - private: NamedAttrList convertedAttr; }; diff --git a/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h b/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h index d487f661a01a..60441f9faaa6 100644 --- a/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h +++ b/mlir/include/mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h @@ -14,19 +14,14 @@ namespace mlir { class ConversionTarget; -class ModuleOp; -template -class OperationPass; +class Pass; class MLIRContext; class TypeConverter; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTASYNCTOLLVM +#define GEN_PASS_DECL_CONVERTASYNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Create a pass to convert Async operations to the LLVM dialect. -std::unique_ptr> createConvertAsyncToLLVMPass(); - /// Populates patterns for async structural type conversions. /// /// A "structural" type conversion is one where the underlying ops are diff --git a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h index 7779321ca46c..1385618c0980 100644 --- a/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h +++ b/mlir/include/mlir/Conversion/ComplexToLLVM/ComplexToLLVM.h @@ -15,7 +15,7 @@ class LLVMTypeConverter; class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTCOMPLEXTOLLVM +#define GEN_PASS_DECL_CONVERTCOMPLEXTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" class ComplexStructBuilder : public StructBuilder { @@ -40,10 +40,6 @@ class ComplexStructBuilder : public StructBuilder { /// Populate the given list with patterns that convert from Complex to LLVM. void populateComplexToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -/// Create a pass to convert Complex operations to the LLVMIR dialect. -std::unique_ptr createConvertComplexToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_COMPLEXTOLLVM_COMPLEXTOLLVM_H_ diff --git a/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h b/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h index b9dfaafcf5a5..9cc867238a00 100644 --- a/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h +++ b/mlir/include/mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h @@ -20,10 +20,11 @@ class LLVMTypeConverter; class RewritePatternSet; class Pass; -#define GEN_PASS_DECL_CONVERTCONTROLFLOWTOLLVM +#define GEN_PASS_DECL_CONVERTCONTROLFLOWTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" namespace cf { + /// Collect the patterns to convert from the ControlFlow dialect to LLVM. The /// conversion patterns capture the LLVMTypeConverter by reference meaning the /// references have to remain alive during the entire pattern lifetime. @@ -36,9 +37,6 @@ void populateControlFlowToLLVMConversionPatterns(LLVMTypeConverter &converter, void populateAssertToLLVMConversionPattern(LLVMTypeConverter &converter, RewritePatternSet &patterns, bool abortOnFailure = true); - -/// Creates a pass to convert the ControlFlow dialect into the LLVMIR dialect. -std::unique_ptr createConvertControlFlowToLLVMPass(); } // namespace cf } // namespace mlir diff --git a/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h b/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h index 701c6acac62f..f6be8a0fcb2b 100644 --- a/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h +++ b/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h @@ -13,20 +13,11 @@ #include namespace mlir { -class LowerToLLVMOptions; -class ModuleOp; -template -class OperationPass; class Pass; -#define GEN_PASS_DECL_CONVERTFUNCTOLLVM +#define GEN_PASS_DECL_CONVERTFUNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Creates a pass to convert the Func dialect into the LLVMIR dialect. -std::unique_ptr> createConvertFuncToLLVMPass(); -std::unique_ptr> -createConvertFuncToLLVMPass(const LowerToLLVMOptions &options); - } // namespace mlir #endif // MLIR_CONVERSION_FUNCTOLLVM_CONVERTFUNCTOLLVMPASS_H_ diff --git a/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h b/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h index 2c73f43c9541..64a9614ee439 100644 --- a/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h +++ b/mlir/include/mlir/Conversion/GPUCommon/GPUCommonPass.h @@ -8,6 +8,7 @@ #ifndef MLIR_CONVERSION_GPUCOMMON_GPUCOMMONPASS_H_ #define MLIR_CONVERSION_GPUCOMMON_GPUCOMMONPASS_H_ +#include "mlir/Dialect/GPU/Transforms/Utils.h" #include "mlir/Support/LLVM.h" #include "llvm/ADT/StringRef.h" #include @@ -27,8 +28,7 @@ class ModuleOp; class Operation; class RewritePatternSet; -template -class OperationPass; +class Pass; namespace gpu { class GPUModuleOp; @@ -47,15 +47,6 @@ using BlobGenerator = using LoweringCallback = std::function( Operation *, llvm::LLVMContext &, StringRef)>; -/// Creates a pass to convert a GPU operations into a sequence of GPU runtime -/// calls. -/// -/// This pass does not generate code to call GPU runtime APIs directly but -/// instead uses a small wrapper library that exports a stable and conveniently -/// typed ABI on top of GPU runtimes such as CUDA or ROCm (HIP). -std::unique_ptr> -createGpuToLLVMConversionPass(bool kernelBarePtrCallConv = false); - /// Collect a set of patterns to convert from the GPU dialect to LLVM and /// populate converter for gpu types. void populateGpuToLLVMConversionPatterns(LLVMTypeConverter &converter, diff --git a/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h b/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h index 240a144bc6cd..a3338ebf982c 100644 --- a/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h +++ b/mlir/include/mlir/Conversion/LLVMCommon/LoweringOptions.h @@ -33,6 +33,7 @@ class LowerToLLVMOptions { LowerToLLVMOptions(MLIRContext *ctx, const DataLayout &dl); bool useBarePtrCallConv = false; + bool useOpaquePointers = false; enum class AllocLowering { /// Use malloc for for heap allocations. diff --git a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h index 2b7735da8466..bf1bd8f7f325 100644 --- a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h +++ b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h @@ -111,6 +111,8 @@ class MemRefDescriptor : public StructBuilder { static unsigned getNumUnpackedValues(MemRefType type); private: + bool useOpaquePointers(); + // Cached index type. Type indexType; }; @@ -194,36 +196,39 @@ class UnrankedMemRefDescriptor : public StructBuilder { /// Builds IR extracting the allocated pointer from the descriptor. static Value allocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, Type elemPtrPtrType); + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType); /// Builds IR inserting the allocated pointer into the descriptor. static void setAllocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, Type elemPtrPtrType, + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType, Value allocatedPtr); /// Builds IR extracting the aligned pointer from the descriptor. static Value alignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType); + LLVM::LLVMPointerType elemPtrType); /// Builds IR inserting the aligned pointer into the descriptor. static void setAlignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, - Value memRefDescPtr, Type elemPtrPtrType, + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType, Value alignedPtr); /// Builds IR extracting the offset from the descriptor. static Value offset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType); + LLVM::LLVMPointerType elemPtrType); /// Builds IR inserting the offset into the descriptor. static void setOffset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType, Value offset); + LLVM::LLVMPointerType elemPtrType, Value offset); /// Builds IR extracting the pointer to the first element of the size array. static Value sizeBasePtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - LLVM::LLVMPointerType elemPtrPtrType); + LLVM::LLVMPointerType elemPtrType); /// Builds IR extracting the size[index] from the descriptor. static Value size(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, diff --git a/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h b/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h index f9c1511948a2..57ef6f48ba09 100644 --- a/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h +++ b/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h @@ -24,6 +24,7 @@ class LowerToLLVMOptions; namespace LLVM { class LLVMDialect; +class LLVMPointerType; } // namespace LLVM /// Conversion from types to the LLVM IR dialect. @@ -119,6 +120,17 @@ class LLVMTypeConverter : public TypeConverter { /// integer type with the size configured for this type converter. Type getIndexType(); + /// Returns true if using opaque pointers was enabled in the lowering options. + bool useOpaquePointers() const { return getOptions().useOpaquePointers; } + + /// Creates an LLVM pointer type with the given element type and address + /// space. + /// This function is meant to be used in code supporting both typed and opaque + /// pointers, as it will create an opaque pointer with the given address space + /// if opaque pointers are enabled in the lowering options. + LLVM::LLVMPointerType getPointerType(Type elementType, + unsigned addressSpace = 0); + /// Gets the bitwidth of the index type when converted to LLVM. unsigned getIndexTypeBitwidth() { return options.getIndexBitwidth(); } diff --git a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h index b75aa4b5e428..72d263c488dc 100644 --- a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h +++ b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h @@ -13,21 +13,16 @@ namespace mlir { class LLVMTypeConverter; class MLIRContext; -class ModuleOp; -template -class OperationPass; +class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTLINALGTOLLVM +#define GEN_PASS_DECL_CONVERTLINALGTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" /// Populate the given list with patterns that convert from Linalg to LLVM. void populateLinalgToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); -/// Create a pass to convert Linalg operations to the LLVMIR dialect. -std::unique_ptr> createConvertLinalgToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_LINALGTOLLVM_LINALGTOLLVM_H_ diff --git a/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h b/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h index 0cb1fe33cadb..d0fc2e390ed7 100644 --- a/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h +++ b/mlir/include/mlir/Conversion/MathToLLVM/MathToLLVM.h @@ -17,13 +17,11 @@ class LLVMTypeConverter; class RewritePatternSet; class Pass; -#define GEN_PASS_DECL_CONVERTMATHTOLLVM +#define GEN_PASS_DECL_CONVERTMATHTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" void populateMathToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -std::unique_ptr createConvertMathToLLVMPass(); } // namespace mlir #endif // MLIR_CONVERSION_MATHTOLLVM_MATHTOLLVM_H diff --git a/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h b/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h index 34d01392e260..943e2108eb97 100644 --- a/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h +++ b/mlir/include/mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h @@ -16,13 +16,13 @@ class Pass; class LLVMTypeConverter; class RewritePatternSet; -#define GEN_PASS_DECL_MEMREFTOLLVMCONVERSIONPASS +#define GEN_PASS_DECL_FINALIZEMEMREFTOLLVMCONVERSIONPASS #include "mlir/Conversion/Passes.h.inc" /// Collect a set of patterns to convert memory-related operations from the /// MemRef dialect to the LLVM dialect. -void populateMemRefToLLVMConversionPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns); +void populateFinalizeMemRefToLLVMConversionPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns); } // namespace mlir #endif // MLIR_CONVERSION_MEMREFTOLLVM_MEMREFTOLLVM_H diff --git a/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h b/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h index 811533d2775d..0cbea0c9ef5b 100644 --- a/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h +++ b/mlir/include/mlir/Conversion/OpenACCToLLVM/ConvertOpenACCToLLVM.h @@ -13,12 +13,10 @@ namespace mlir { class LLVMTypeConverter; -class ModuleOp; -template -class OperationPass; +class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTOPENACCTOLLVM +#define GEN_PASS_DECL_CONVERTOPENACCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" static constexpr unsigned kPtrBasePosInDataDescriptor = 0; @@ -69,10 +67,6 @@ class DataDescriptor : public StructBuilder { /// Collect the patterns to convert from the OpenACC dialect LLVMIR dialect. void populateOpenACCToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -/// Create a pass to convert the OpenACC dialect into the LLVMIR dialect. -std::unique_ptr> createConvertOpenACCToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_OPENACCTOLLVM_CONVERTOPENACCTOLLVM_H diff --git a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h index 6827055b1c10..2ed077f8f83e 100644 --- a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h +++ b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h @@ -14,12 +14,10 @@ namespace mlir { class LLVMTypeConverter; class ConversionTarget; class MLIRContext; -class ModuleOp; -template -class OperationPass; +class Pass; class RewritePatternSet; -#define GEN_PASS_DECL_CONVERTOOPENMPTOLLVM +#define GEN_PASS_DECL_CONVERTOPENMPTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" /// Configure dynamic conversion legality of regionless operations from OpenMP @@ -30,10 +28,6 @@ void configureOpenMPToLLVMConversionLegality(ConversionTarget &target, /// Populate the given list with patterns that convert from OpenMP to LLVM. void populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns); - -/// Create a pass to convert OpenMP operations to the LLVMIR dialect. -std::unique_ptr> createConvertOpenMPToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_OPENMPTOLLVM_CONVERTOPENMPTOLLVM_H diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index fc03d0037f1c..d7d60126ce78 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -143,14 +143,13 @@ def ConvertArmNeon2dToIntr : Pass<"arm-neon-2d-to-intr"> { // AsyncToLLVM //===----------------------------------------------------------------------===// -def ConvertAsyncToLLVM : Pass<"convert-async-to-llvm", "ModuleOp"> { +def ConvertAsyncToLLVMPass : Pass<"convert-async-to-llvm", "ModuleOp"> { let summary = "Convert the operations from the async dialect into the LLVM " "dialect"; let description = [{ Convert `async.execute` operations to LLVM coroutines and use async runtime API to execute them. }]; - let constructor = "mlir::createConvertAsyncToLLVMPass()"; let dependentDialects = [ "arith::ArithDialect", "async::AsyncDialect", @@ -198,9 +197,8 @@ def ConvertBufferizationToMemRef : Pass<"convert-bufferization-to-memref"> { // ComplexToLLVM //===----------------------------------------------------------------------===// -def ConvertComplexToLLVM : Pass<"convert-complex-to-llvm"> { +def ConvertComplexToLLVMPass : Pass<"convert-complex-to-llvm"> { let summary = "Convert Complex dialect to LLVM dialect"; - let constructor = "mlir::createConvertComplexToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -233,7 +231,7 @@ def ConvertComplexToStandard : Pass<"convert-complex-to-standard"> { // ControlFlowToLLVM //===----------------------------------------------------------------------===// -def ConvertControlFlowToLLVM : Pass<"convert-cf-to-llvm", "ModuleOp"> { +def ConvertControlFlowToLLVMPass : Pass<"convert-cf-to-llvm", "ModuleOp"> { let summary = "Convert ControlFlow operations to the LLVM dialect"; let description = [{ Convert ControlFlow operations into LLVM IR dialect operations. @@ -242,7 +240,6 @@ def ConvertControlFlowToLLVM : Pass<"convert-cf-to-llvm", "ModuleOp"> { IR dialect operations, the pass will fail. Any LLVM IR operations or types already present in the IR will be kept as is. }]; - let constructor = "mlir::cf::createConvertControlFlowToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; let options = [ Option<"indexBitwidth", "index-bitwidth", "unsigned", @@ -271,7 +268,7 @@ def ConvertControlFlowToSPIRV : Pass<"convert-cf-to-spirv"> { // FuncToLLVM //===----------------------------------------------------------------------===// -def ConvertFuncToLLVM : Pass<"convert-func-to-llvm", "ModuleOp"> { +def ConvertFuncToLLVMPass : Pass<"convert-func-to-llvm", "ModuleOp"> { let summary = "Convert from the Func dialect to the LLVM dialect"; let description = [{ Convert Func dialect operations into the LLVM IR dialect operations. @@ -295,7 +292,6 @@ def ConvertFuncToLLVM : Pass<"convert-func-to-llvm", "ModuleOp"> { returns are updated accordingly. Block argument types are updated to use LLVM IR types. }]; - let constructor = "mlir::createConvertFuncToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; let options = [ Option<"useBarePtrCallConv", "use-bare-ptr-memref-call-conv", "bool", @@ -334,16 +330,48 @@ def ConvertFuncToSPIRV : Pass<"convert-func-to-spirv"> { def GpuToLLVMConversionPass : Pass<"gpu-to-llvm", "ModuleOp"> { let summary = "Convert GPU dialect to LLVM dialect with GPU runtime calls"; - let constructor = "mlir::createGpuToLLVMConversionPass()"; + + let description = [{ + Creates a pass to convert a GPU operations into a sequence of GPU runtime + calls. + + This pass does not generate code to call GPU runtime APIs directly but + instead uses a small wrapper library that exports a stable and conveniently + typed ABI on top of GPU runtimes such as CUDA or ROCm (HIP). + }]; + + let options = [ + Option<"kernelBarePtrCallConv", "use-bare-pointers-for-kernels", "bool", + /*default=*/"false", + "Use bare pointers to pass memref arguments to kernels. " + "The kernel must use the same setting for this option." + >, + Option<"gpuBinaryAnnotation", "gpu-binary-annotation", "std::string", + /*default=*/"gpu::getDefaultGpuBinaryAnnotation()", + "Annotation attribute string for GPU binary" + >, + ]; + let dependentDialects = [ "LLVM::LLVMDialect", "memref::MemRefDialect", ]; } -def LowerHostCodeToLLVM : Pass<"lower-host-to-llvm", "ModuleOp"> { +def LowerHostCodeToLLVMPass : Pass<"lower-host-to-llvm", "ModuleOp"> { let summary = "Lowers the host module code and `gpu.launch_func` to LLVM"; - let constructor = "mlir::createLowerHostCodeToLLVMPass()"; + + let description = [{ + Creates a pass to emulate `gpu.launch_func` call in LLVM dialect and lower + the host module code to LLVM. + + This transformation creates a sequence of global variables that are later + linked to the variables in the kernel module, and a series of copies to/from + them to emulate the memory transfer from the host or to the device sides. It + also converts the remaining Arithmetic, Func, and MemRef dialects into LLVM + dialect, emitting C wrappers. + }]; + let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -476,10 +504,9 @@ def ConvertIndexToLLVMPass : Pass<"convert-index-to-llvm"> { // LinalgToLLVM //===----------------------------------------------------------------------===// -def ConvertLinalgToLLVM : Pass<"convert-linalg-to-llvm", "ModuleOp"> { +def ConvertLinalgToLLVMPass : Pass<"convert-linalg-to-llvm", "ModuleOp"> { let summary = "Convert the operations from the linalg dialect into the LLVM " "dialect"; - let constructor = "mlir::createConvertLinalgToLLVMPass()"; let dependentDialects = ["scf::SCFDialect", "LLVM::LLVMDialect"]; } @@ -515,12 +542,11 @@ def ConvertMathToLibm : Pass<"convert-math-to-libm", "ModuleOp"> { // MathToLLVM //===----------------------------------------------------------------------===// -def ConvertMathToLLVM : Pass<"convert-math-to-llvm"> { +def ConvertMathToLLVMPass : Pass<"convert-math-to-llvm"> { let summary = "Convert Math dialect to LLVM dialect"; let description = [{ This pass converts supported Math ops to LLVM dialect intrinsics. }]; - let constructor = "mlir::createConvertMathToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -564,9 +590,13 @@ def ConvertMathToFuncs : Pass<"convert-math-to-funcs", "ModuleOp"> { // MemRefToLLVM //===----------------------------------------------------------------------===// -def MemRefToLLVMConversionPass : Pass<"convert-memref-to-llvm", "ModuleOp"> { - let summary = "Convert operations from the MemRef dialect to the LLVM " - "dialect"; +def FinalizeMemRefToLLVMConversionPass : + Pass<"finalize-memref-to-llvm", "ModuleOp"> { + let summary = "Finalize the conversion of the operations from the MemRef " + "dialect to the LLVM dialect." + "This conversion will not convert some complex MemRef " + "operations. Make sure to run `expand-strided-metadata` " + "beforehand for these."; let dependentDialects = ["LLVM::LLVMDialect"]; let options = [ Option<"useAlignedAlloc", "use-aligned-alloc", "bool", /*default=*/"false", @@ -578,7 +608,10 @@ def MemRefToLLVMConversionPass : Pass<"convert-memref-to-llvm", "ModuleOp"> { "bool", /*default=*/"false", "Use generic allocation and deallocation functions instead of the " - "classic 'malloc', 'aligned_alloc' and 'free' functions"> + "classic 'malloc', 'aligned_alloc' and 'free' functions">, + Option<"useOpaquePointers", "use-opaque-pointers", "bool", + /*default=*/"false", "Generate LLVM IR using opaque pointers " + "instead of typed pointers"> ]; } @@ -637,9 +670,8 @@ def ConvertOpenACCToSCF : Pass<"convert-openacc-to-scf", "ModuleOp"> { // OpenACCToLLVM //===----------------------------------------------------------------------===// -def ConvertOpenACCToLLVM : Pass<"convert-openacc-to-llvm", "ModuleOp"> { +def ConvertOpenACCToLLVMPass : Pass<"convert-openacc-to-llvm", "ModuleOp"> { let summary = "Convert the OpenACC ops to LLVM dialect"; - let constructor = "mlir::createConvertOpenACCToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -647,9 +679,8 @@ def ConvertOpenACCToLLVM : Pass<"convert-openacc-to-llvm", "ModuleOp"> { // OpenMPToLLVM //===----------------------------------------------------------------------===// -def ConvertOpenMPToLLVM : Pass<"convert-openmp-to-llvm", "ModuleOp"> { +def ConvertOpenMPToLLVMPass : Pass<"convert-openmp-to-llvm", "ModuleOp"> { let summary = "Convert the OpenMP ops to OpenMP ops with LLVM dialect"; - let constructor = "mlir::createConvertOpenMPToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -784,13 +815,12 @@ def ConvertShapeConstraints : Pass<"convert-shape-constraints"> { // SPIRVToLLVM //===----------------------------------------------------------------------===// -def ConvertSPIRVToLLVM : Pass<"convert-spirv-to-llvm", "ModuleOp"> { +def ConvertSPIRVToLLVMPass : Pass<"convert-spirv-to-llvm", "ModuleOp"> { let summary = "Convert SPIR-V dialect to LLVM dialect"; let description = [{ See https://mlir.llvm.org/docs/SPIRVToLLVMDialectConversion/ for more details. }]; - let constructor = "mlir::createConvertSPIRVToLLVMPass()"; let dependentDialects = ["LLVM::LLVMDialect"]; } @@ -958,7 +988,7 @@ def ConvertVectorToSCF : Pass<"convert-vector-to-scf"> { // VectorToLLVM //===----------------------------------------------------------------------===// -def ConvertVectorToLLVM : Pass<"convert-vector-to-llvm", "ModuleOp"> { +def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm", "ModuleOp"> { let summary = "Lower the operations from the vector dialect into the LLVM " "dialect"; let description = [{ @@ -971,7 +1001,6 @@ def ConvertVectorToLLVM : Pass<"convert-vector-to-llvm", "ModuleOp"> { architectural-neutral vector dialect lowering. }]; - let constructor = "mlir::createConvertVectorToLLVMPass()"; // Override explicitly in C++ to allow conditional dialect dependence. // let dependentDialects; let options = [ diff --git a/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h b/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h index 1e6b322512c6..c845a745e89d 100644 --- a/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h +++ b/mlir/include/mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h @@ -16,27 +16,12 @@ #include namespace mlir { -class ModuleOp; -template -class OperationPass; +class Pass; -#define GEN_PASS_DECL_LOWERHOSTCODETOLLVM -#define GEN_PASS_DECL_CONVERTSPIRVTOLLVM +#define GEN_PASS_DECL_LOWERHOSTCODETOLLVMPASS +#define GEN_PASS_DECL_CONVERTSPIRVTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Creates a pass to emulate `gpu.launch_func` call in LLVM dialect and lower -/// the host module code to LLVM. -/// -/// This transformation creates a sequence of global variables that are later -/// linked to the variables in the kernel module, and a series of copies to/from -/// them to emulate the memory transfer from the host or to the device sides. It -/// also converts the remaining Arithmetic, Func, and MemRef dialects into LLVM -/// dialect, emitting C wrappers. -std::unique_ptr> createLowerHostCodeToLLVMPass(); - -/// Creates a pass to convert SPIR-V operations to the LLVMIR dialect. -std::unique_ptr> createConvertSPIRVToLLVMPass(); - } // namespace mlir #endif // MLIR_CONVERSION_SPIRVTOLLVM_SPIRVTOLLVMPASS_H diff --git a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h index 0931af0774fe..0d5d1c8b5ffe 100644 --- a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h +++ b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h @@ -12,53 +12,11 @@ namespace mlir { class LLVMTypeConverter; -class ModuleOp; -template -class OperationPass; +class Pass; -#define GEN_PASS_DECL_CONVERTVECTORTOLLVM +#define GEN_PASS_DECL_CONVERTVECTORTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" -/// Options to control Vector to LLVM lowering. -/// -/// This should kept in sync with VectorToLLVM options defined for the -/// ConvertVectorToLLVM pass in include/mlir/Conversion/Passes.td -struct LowerVectorToLLVMOptions { - LowerVectorToLLVMOptions() {} - - LowerVectorToLLVMOptions &enableReassociateFPReductions(bool b = true) { - reassociateFPReductions = b; - return *this; - } - LowerVectorToLLVMOptions &enableIndexOptimizations(bool b = true) { - force32BitVectorIndices = b; - return *this; - } - LowerVectorToLLVMOptions &enableArmNeon(bool b = true) { - armNeon = b; - return *this; - } - LowerVectorToLLVMOptions &enableArmSVE(bool b = true) { - armSVE = b; - return *this; - } - LowerVectorToLLVMOptions &enableAMX(bool b = true) { - amx = b; - return *this; - } - LowerVectorToLLVMOptions &enableX86Vector(bool b = true) { - x86Vector = b; - return *this; - } - - bool reassociateFPReductions{false}; - bool force32BitVectorIndices{true}; - bool armNeon{false}; - bool armSVE{false}; - bool amx{false}; - bool x86Vector{false}; -}; - /// Collect a set of patterns to convert from Vector contractions to LLVM Matrix /// Intrinsics. To lower to assembly, the LLVM flag -lower-matrix-intrinsics /// will be needed when invoking LLVM. @@ -70,10 +28,6 @@ void populateVectorToLLVMConversionPatterns( LLVMTypeConverter &converter, RewritePatternSet &patterns, bool reassociateFPReductions = false, bool force32BitVectorIndices = false); -/// Create a pass to convert vector operations to the LLVMIR dialect. -std::unique_ptr> createConvertVectorToLLVMPass( - const LowerVectorToLLVMOptions &options = LowerVectorToLLVMOptions()); - } // namespace mlir #endif // MLIR_CONVERSION_VECTORTOLLVM_CONVERTVECTORTOLLVM_H_ diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h index 594d8be0838b..657fb6dd3eea 100644 --- a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h +++ b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h @@ -13,6 +13,7 @@ #ifndef MLIR_DIALECT_GPU_TRANSFORMS_PASSES_H_ #define MLIR_DIALECT_GPU_TRANSFORMS_PASSES_H_ +#include "Utils.h" #include "mlir/Dialect/GPU/IR/GPUDialect.h" #include "mlir/Pass/Pass.h" #include diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h b/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h index e72d2bc41c52..a426bee7686d 100644 --- a/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h +++ b/mlir/include/mlir/Dialect/GPU/Transforms/Utils.h @@ -15,6 +15,8 @@ #include "mlir/Support/LLVM.h" +#include + namespace mlir { struct LogicalResult; class Operation; @@ -23,6 +25,9 @@ class Value; namespace gpu { class GPUFuncOp; class LaunchOp; + +/// Returns the default annotation name for GPU binary blobs. +std::string getDefaultGpuBinaryAnnotation(); } // namespace gpu /// Get a gpu.func created from outlining the region of a gpu.launch op with the diff --git a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h index 3421e9bb67b6..76838ddf7a63 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h +++ b/mlir/include/mlir/Dialect/LLVMIR/FunctionCallUtils.h @@ -36,20 +36,24 @@ LLVM::LLVMFuncOp lookupOrCreatePrintI64Fn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintU64Fn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintF32Fn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintF64Fn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreatePrintStrFn(ModuleOp moduleOp); +LLVM::LLVMFuncOp lookupOrCreatePrintStrFn(ModuleOp moduleOp, + bool opaquePointers); LLVM::LLVMFuncOp lookupOrCreatePrintOpenFn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintCloseFn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintCommaFn(ModuleOp moduleOp); LLVM::LLVMFuncOp lookupOrCreatePrintNewlineFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreateMallocFn(ModuleOp moduleOp, Type indexType); -LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, - Type indexType); -LLVM::LLVMFuncOp lookupOrCreateFreeFn(ModuleOp moduleOp); -LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(ModuleOp moduleOp, - Type indexType); +LLVM::LLVMFuncOp lookupOrCreateMallocFn(ModuleOp moduleOp, Type indexType, + bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, Type indexType, + bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateFreeFn(ModuleOp moduleOp, bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateGenericAllocFn(ModuleOp moduleOp, Type indexType, + bool opaquePointers); LLVM::LLVMFuncOp lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp, - Type indexType); -LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(ModuleOp moduleOp); + Type indexType, + bool opaquePointers); +LLVM::LLVMFuncOp lookupOrCreateGenericFreeFn(ModuleOp moduleOp, + bool opaquePointers); LLVM::LLVMFuncOp lookupOrCreateMemRefCopyFn(ModuleOp moduleOp, Type indexType, Type unrankedDescriptorType); diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index 96936c4dcdc4..26a469dc79da 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -50,38 +50,177 @@ def LinkageAttr : LLVM_Attr<"Linkage", "linkage"> { } //===----------------------------------------------------------------------===// -// LoopOptionsAttr +// Loop Attributes //===----------------------------------------------------------------------===// -def LoopOptionsAttr : LLVM_Attr<"LoopOptions", "loopopts"> { +def LoopVectorizeAttr : LLVM_Attr<"LoopVectorize", "loop_vectorize"> { let description = [{ - This attributes encapsulates "loop options". It is means to decorate - branches that are "latches" (loop backedges) and maps to the `!llvm.loop` - metadatas: https://llvm.org/docs/LangRef.html#llvm-loop - It store the options as a pair in a sorted array and expose - APIs to retrieve the value for each option with a stronger type (bool for - example). + This attribute defines vectorization specific loop annotations that map to + the "!llvm.loop.vectorize" metadata. }]; let parameters = (ins - ArrayRefParameter<"std::pair", "">:$options + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$predicateEnable, + OptionalParameter<"BoolAttr">:$scalableEnable, + OptionalParameter<"IntegerAttr">:$width, + OptionalParameter<"LoopAnnotationAttr">:$followupVectorized, + OptionalParameter<"LoopAnnotationAttr">:$followupEpilogue, + OptionalParameter<"LoopAnnotationAttr">:$followupAll ); - let extraClassDeclaration = [{ - using OptionValuePair = std::pair; - using OptionsArray = ArrayRef>; - std::optional disableUnroll(); - std::optional disableLICM(); - std::optional interleaveCount(); + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopInterleaveAttr : LLVM_Attr<"LoopInterleave", "loop_interleave"> { + let description = [{ + This attribute defines interleaving specific loop annotations that map to + the "!llvm.loop.interleave" metadata. }]; - let builders = [ - /// Build the LoopOptions Attribute from a sorted array of individual options. - AttrBuilder<(ins "ArrayRef>":$sortedOptions)>, - AttrBuilder<(ins "LoopOptionsAttrBuilder &":$optionBuilders)> - ]; - let hasCustomAssemblyFormat = 1; - let skipDefaultBuilders = 1; + let parameters = (ins + "IntegerAttr":$count + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollAttr : LLVM_Attr<"LoopUnroll", "loop_unroll"> { + let description = [{ + This attribute defines unrolling specific loop annotations that map to + the "!llvm.loop.unroll" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"BoolAttr">:$runtimeDisable, + OptionalParameter<"BoolAttr">:$full, + OptionalParameter<"LoopAnnotationAttr">:$followupUnrolled, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainder, + OptionalParameter<"LoopAnnotationAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnrollAndJamAttr : LLVM_Attr<"LoopUnrollAndJam", "loop_unroll_and_jam"> { + let description = [{ + This attribute defines "unroll and jam" specific loop annotations that map to + the "!llvm.loop.unroll_and_jam" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$count, + OptionalParameter<"LoopAnnotationAttr">:$followupOuter, + OptionalParameter<"LoopAnnotationAttr">:$followupInner, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainderOuter, + OptionalParameter<"LoopAnnotationAttr">:$followupRemainderInner, + OptionalParameter<"LoopAnnotationAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopLICMAttr : LLVM_Attr<"LoopLICM", "loop_licm"> { + let description = [{ + This attribute encapsulates loop invariant code motion (licm) specific loop + annotations. The fields correspond to the "!llvm.licm.disable" and the + "!llvm.loop.licm_versioning.disable" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$versioningDisable + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopDistributeAttr : LLVM_Attr<"LoopDistribute", "loop_distribute"> { + let description = [{ + This attribute defines distribution specific loop annotations that map to + the "!llvm.loop.distribute" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"LoopAnnotationAttr">:$followupCoincident, + OptionalParameter<"LoopAnnotationAttr">:$followupSequential, + OptionalParameter<"LoopAnnotationAttr">:$followupFallback, + OptionalParameter<"LoopAnnotationAttr">:$followupAll + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopPipelineAttr : LLVM_Attr<"LoopPipeline", "loop_pipeline"> { + let description = [{ + This attribute defines pipelining specific loop annotations that map to + the "!llvm.loop.pipeline" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"IntegerAttr">:$initiationinterval + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopPeeledAttr : LLVM_Attr<"LoopPeeled", "loop_peeled"> { + let description = [{ + This attribute defines pipelining specific loop annotations that map to + the "!llvm.loop.peeled" metadata. + }]; + + let parameters = (ins + OptionalParameter<"IntegerAttr">:$count + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopUnswitchAttr : LLVM_Attr<"LoopUnswitch", "loop_unswitch"> { + let description = [{ + This attribute defines pipelining specific loop annotations that map to + the "!llvm.loop.unswitch" metadata. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$partialDisable + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + +def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> { + let description = [{ + This attributes encapsulates "loop metadata". It is meant to decorate + branches that are "latches" (loop backedges) and maps to the `!llvm.loop` + metadatas: https://llvm.org/docs/LangRef.html#llvm-loop + It stores annotations in attribute parameters and groups related options in + nested attributes to provide structured access. + }]; + + let parameters = (ins + OptionalParameter<"BoolAttr">:$disableNonforced, + OptionalParameter<"LoopVectorizeAttr">:$vectorize, + OptionalParameter<"LoopInterleaveAttr">:$interleave, + OptionalParameter<"LoopUnrollAttr">:$unroll, + OptionalParameter<"LoopUnrollAndJamAttr">:$unrollAndJam, + OptionalParameter<"LoopLICMAttr">:$licm, + OptionalParameter<"LoopDistributeAttr">:$distribute, + OptionalParameter<"LoopPipelineAttr">:$pipeline, + OptionalParameter<"LoopPeeledAttr">:$peeled, + OptionalParameter<"LoopUnswitchAttr">:$unswitch, + OptionalParameter<"BoolAttr">:$mustProgress, + OptionalParameter<"BoolAttr">:$isVectorized, + OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses + ); + + let assemblyFormat = "`<` struct(params) `>`"; } //===----------------------------------------------------------------------===// @@ -123,11 +262,11 @@ def LLVM_DITagParameter : LLVM_DIParameter< >; //===----------------------------------------------------------------------===// -// DIVoidResultTypeAttr +// DINullTypeAttr //===----------------------------------------------------------------------===// -def LLVM_DIVoidResultTypeAttr : LLVM_Attr<"DIVoidResultType", "di_void_result_type", - /*traits=*/[], "DITypeAttr"> { +def LLVM_DINullTypeAttr : LLVM_Attr<"DINullType", "di_null_type", + /*traits=*/[], "DITypeAttr"> { let parameters = (ins); } @@ -166,7 +305,7 @@ def LLVM_DICompileUnitAttr : LLVM_Attr<"DICompileUnit", "di_compile_unit", [ let parameters = (ins LLVM_DILanguageParameter:$sourceLanguage, "DIFileAttr":$file, - "StringAttr":$producer, + OptionalParameter<"StringAttr">:$producer, "bool":$isOptimized, "DIEmissionKind":$emissionKind ); @@ -182,7 +321,7 @@ def LLVM_DICompositeTypeAttr : LLVM_Attr<"DICompositeType", "di_composite_type", ], "DITypeAttr"> { let parameters = (ins LLVM_DITagParameter:$tag, - "StringAttr":$name, + OptionalParameter<"StringAttr">:$name, OptionalParameter<"DIFileAttr">:$file, OptionalParameter<"uint32_t">:$line, OptionalParameter<"DIScopeAttr">:$scope, @@ -205,7 +344,7 @@ def LLVM_DIDerivedTypeAttr : LLVM_Attr<"DIDerivedType", "di_derived_type", [ let parameters = (ins LLVM_DITagParameter:$tag, OptionalParameter<"StringAttr">:$name, - "DITypeAttr":$baseType, + OptionalParameter<"DITypeAttr">:$baseType, OptionalParameter<"uint64_t">:$sizeInBits, OptionalParameter<"uint32_t">:$alignInBits, OptionalParameter<"uint64_t">:$offsetInBits @@ -245,7 +384,7 @@ def LLVM_DILexicalBlockAttr : LLVM_Attr<"DILexicalBlock", "di_lexical_block", [ "DIScopeAttr":$scope, "DIFileAttr":$file, "unsigned":$line, "unsigned":$column ), [{ - return $_get(file.getContext(), scope, file, line, column); + return $_get(scope.getContext(), scope, file, line, column); }]> ]; let assemblyFormat = "`<` struct(params) `>`"; @@ -267,7 +406,7 @@ def LLVM_DILexicalBlockFile : LLVM_Attr<"DILexicalBlockFile", "di_lexical_block_ AttrBuilderWithInferredContext<(ins "DIScopeAttr":$scope, "DIFileAttr":$file, "unsigned":$discriminator ), [{ - return $_get(file.getContext(), scope, file, discriminator); + return $_get(scope.getContext(), scope, file, discriminator); }]> ]; let assemblyFormat = "`<` struct(params) `>`"; @@ -282,7 +421,7 @@ def LLVM_DILocalVariableAttr : LLVM_Attr<"DILocalVariable", "di_local_variable", ], "DINodeAttr"> { let parameters = (ins "DIScopeAttr":$scope, - "StringAttr":$name, + OptionalParameter<"StringAttr">:$name, OptionalParameter<"DIFileAttr">:$file, OptionalParameter<"unsigned">:$line, OptionalParameter<"unsigned">:$arg, @@ -295,7 +434,7 @@ def LLVM_DILocalVariableAttr : LLVM_Attr<"DILocalVariable", "di_local_variable", "unsigned":$line, "unsigned":$arg, "unsigned":$alignInBits, "DITypeAttr":$type ), [{ - MLIRContext *ctx = file.getContext(); + MLIRContext *ctx = scope.getContext(); return $_get(ctx, scope, StringAttr::get(ctx, name), file, line, arg, alignInBits, type); }]> @@ -311,7 +450,7 @@ def LLVM_DISubprogramAttr : LLVM_Attr<"DISubprogram", "di_subprogram", [ SubElementAttrInterface ], "DIScopeAttr"> { let parameters = (ins - "DICompileUnitAttr":$compileUnit, + OptionalParameter<"DICompileUnitAttr">:$compileUnit, "DIScopeAttr":$scope, "StringAttr":$name, OptionalParameter<"StringAttr">:$linkageName, @@ -338,6 +477,21 @@ def LLVM_DISubprogramAttr : LLVM_Attr<"DISubprogram", "di_subprogram", [ let assemblyFormat = "`<` struct(params) `>`"; } +//===----------------------------------------------------------------------===// +// DINamespaceAttr +//===----------------------------------------------------------------------===// + +def LLVM_DINamespaceAttr : LLVM_Attr<"DINamespace", "di_namespace", + /*traits=*/[], "DIScopeAttr"> { + let parameters = (ins + "StringAttr":$name, + OptionalParameter<"DIScopeAttr">:$scope, + "bool":$exportSymbols + ); + + let assemblyFormat = "`<` struct(params) `>`"; +} + //===----------------------------------------------------------------------===// // DISubrangeAttr //===----------------------------------------------------------------------===// @@ -370,7 +524,6 @@ def LLVM_DISubroutineTypeAttr : LLVM_Attr<"DISubroutineType", "di_subroutine_typ }]> ]; let assemblyFormat = "`<` struct(params) `>`"; - let genVerifyDecl = 1; } //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h index d6619ac8c9b9..abbfcf415c54 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h @@ -22,7 +22,6 @@ namespace mlir { namespace LLVM { -class LoopOptionsAttrBuilder; /// This class represents the base attribute for all debug info attributes. class DINodeAttr : public Attribute { @@ -74,54 +73,4 @@ using linkage::Linkage; #define GET_ATTRDEF_CLASSES #include "mlir/Dialect/LLVMIR/LLVMOpsAttrDefs.h.inc" -namespace mlir { -namespace LLVM { - -/// Builder class for LoopOptionsAttr. This helper class allows to progressively -/// build a LoopOptionsAttr one option at a time, and pay the price of attribute -/// creation once all the options are in place. -class LoopOptionsAttrBuilder { -public: - /// Construct a empty builder. - LoopOptionsAttrBuilder() = default; - - /// Construct a builder with an initial list of options from an existing - /// LoopOptionsAttr. - LoopOptionsAttrBuilder(LoopOptionsAttr attr); - - /// Set the `disable_licm` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableLICM(std::optional value); - - /// Set the `interleave_count` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setInterleaveCount(std::optional count); - - /// Set the `disable_unroll` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisableUnroll(std::optional value); - - /// Set the `disable_pipeline` option to the provided value. If no value - /// is provided the option is deleted. - LoopOptionsAttrBuilder &setDisablePipeline(std::optional value); - - /// Set the `pipeline_initiation_interval` option to the provided value. - /// If no value is provided the option is deleted. - LoopOptionsAttrBuilder & - setPipelineInitiationInterval(std::optional count); - - /// Returns true if any option has been set. - bool empty() { return options.empty(); } - -private: - template - LoopOptionsAttrBuilder &setOption(LoopOptionCase tag, std::optional value); - - friend class LoopOptionsAttr; - SmallVector options; -}; - -} // namespace LLVM -} // namespace mlir - #endif // MLIR_DIALECT_LLVMIR_LLVMATTRS_H_ diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td index 17b372f3774e..7fd27b182005 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -33,42 +33,64 @@ def AsmATTOrIntel : LLVM_EnumAttr< // Atomic Operations //===----------------------------------------------------------------------===// -def AtomicBinOpXchg : I64EnumAttrCase<"xchg", 0>; -def AtomicBinOpAdd : I64EnumAttrCase<"add", 1>; -def AtomicBinOpSub : I64EnumAttrCase<"sub", 2>; -def AtomicBinOpAnd : I64EnumAttrCase<"_and", 3>; -def AtomicBinOpNand : I64EnumAttrCase<"nand", 4>; -def AtomicBinOpOr : I64EnumAttrCase<"_or", 5>; -def AtomicBinOpXor : I64EnumAttrCase<"_xor", 6>; -def AtomicBinOpMax : I64EnumAttrCase<"max", 7>; -def AtomicBinOpMin : I64EnumAttrCase<"min", 8>; -def AtomicBinOpUMax : I64EnumAttrCase<"umax", 9>; -def AtomicBinOpUMin : I64EnumAttrCase<"umin", 10>; -def AtomicBinOpFAdd : I64EnumAttrCase<"fadd", 11>; -def AtomicBinOpFSub : I64EnumAttrCase<"fsub", 12>; -def AtomicBinOp : I64EnumAttr< +def AtomicBinOpXchg : LLVM_EnumAttrCase<"xchg", "xchg", "Xchg", 0>; +def AtomicBinOpAdd : LLVM_EnumAttrCase<"add", "add", "Add", 1>; +def AtomicBinOpSub : LLVM_EnumAttrCase<"sub", "sub", "Sub", 2>; +def AtomicBinOpAnd : LLVM_EnumAttrCase<"_and", "_and", "And", 3>; +def AtomicBinOpNand : LLVM_EnumAttrCase<"nand", "nand", "Nand", 4>; +def AtomicBinOpOr : LLVM_EnumAttrCase<"_or", "_or", "Or", 5>; +def AtomicBinOpXor : LLVM_EnumAttrCase<"_xor", "_xor", "Xor", 6>; +def AtomicBinOpMax : LLVM_EnumAttrCase<"max", "max", "Max", 7>; +def AtomicBinOpMin : LLVM_EnumAttrCase<"min", "min", "Min", 8>; +def AtomicBinOpUMax : LLVM_EnumAttrCase<"umax", "umax", "UMax", 9>; +def AtomicBinOpUMin : LLVM_EnumAttrCase<"umin", "umin", "UMin", 10>; +def AtomicBinOpFAdd : LLVM_EnumAttrCase<"fadd", "fadd", "FAdd", 11>; +def AtomicBinOpFSub : LLVM_EnumAttrCase<"fsub", "fsub", "FSub", 12>; +def AtomicBinOpFMax : LLVM_EnumAttrCase<"fmax", "fmax", "FMax", 13>; +def AtomicBinOpFMin : LLVM_EnumAttrCase<"fmin", "fmin", "FMin", 14>; +def AtomicBinOpUIncWrap : LLVM_EnumAttrCase<"uinc_wrap", + "uinc_wrap", "UIncWrap", 15>; +def AtomicBinOpUDecWrap : LLVM_EnumAttrCase<"udec_wrap", + "udec_wrap", "UDecWrap", 16>; + +// A sentinel value that has no MLIR counterpart. +def AtomicBadBinOp : LLVM_EnumAttrCase<"", "", "BAD_BINOP", 0>; + +def AtomicBinOp : LLVM_EnumAttr< "AtomicBinOp", + "::llvm::AtomicRMWInst::BinOp", "llvm.atomicrmw binary operations", [AtomicBinOpXchg, AtomicBinOpAdd, AtomicBinOpSub, AtomicBinOpAnd, AtomicBinOpNand, AtomicBinOpOr, AtomicBinOpXor, AtomicBinOpMax, AtomicBinOpMin, AtomicBinOpUMax, AtomicBinOpUMin, AtomicBinOpFAdd, - AtomicBinOpFSub]> { + AtomicBinOpFSub, AtomicBinOpFMax, AtomicBinOpFMin, AtomicBinOpUIncWrap, + AtomicBinOpUDecWrap], + [AtomicBadBinOp]> { let cppNamespace = "::mlir::LLVM"; } -def AtomicOrderingNotAtomic : I64EnumAttrCase<"not_atomic", 0>; -def AtomicOrderingUnordered : I64EnumAttrCase<"unordered", 1>; -def AtomicOrderingMonotonic : I64EnumAttrCase<"monotonic", 2>; -def AtomicOrderingAcquire : I64EnumAttrCase<"acquire", 4>; -def AtomicOrderingRelease : I64EnumAttrCase<"release", 5>; -def AtomicOrderingAcquireRelease : I64EnumAttrCase<"acq_rel", 6>; -def AtomicOrderingSequentiallyConsistent : I64EnumAttrCase<"seq_cst", 7>; -def AtomicOrdering : I64EnumAttr< +def AtomicOrderingNotAtomic : LLVM_EnumAttrCase<"not_atomic", + "not_atomic", "NotAtomic", 0>; +def AtomicOrderingUnordered : LLVM_EnumAttrCase<"unordered", + "unordered", "Unordered", 1>; +def AtomicOrderingMonotonic : LLVM_EnumAttrCase<"monotonic", + "monotonic", "Monotonic", 2>; +def AtomicOrderingAcquire : LLVM_EnumAttrCase<"acquire", + "acquire", "Acquire", 4>; +def AtomicOrderingRelease : LLVM_EnumAttrCase<"release", + "release", "Release", 5>; +def AtomicOrderingAcquireRelease : + LLVM_EnumAttrCase<"acq_rel", "acq_rel", "AcquireRelease", 6>; +def AtomicOrderingSequentiallyConsistent : + LLVM_EnumAttrCase<"seq_cst", "seq_cst", "SequentiallyConsistent", 7>; +def AtomicOrdering : LLVM_EnumAttr< "AtomicOrdering", + "::llvm::AtomicOrdering", "Atomic ordering for LLVM's memory model", [AtomicOrderingNotAtomic, AtomicOrderingUnordered, AtomicOrderingMonotonic, AtomicOrderingAcquire, AtomicOrderingRelease, AtomicOrderingAcquireRelease, - AtomicOrderingSequentiallyConsistent]> { + AtomicOrderingSequentiallyConsistent + ]> { let cppNamespace = "::mlir::LLVM"; } @@ -380,59 +402,77 @@ def FastmathFlags : I32BitEnumAttr< } //===----------------------------------------------------------------------===// -// FCmp Predicates +// FCmp and ICmp Predicates //===----------------------------------------------------------------------===// -// Predicate for float comparisons -def FCmpPredicateFALSE : I64EnumAttrCase<"_false", 0>; -def FCmpPredicateOEQ : I64EnumAttrCase<"oeq", 1>; -def FCmpPredicateOGT : I64EnumAttrCase<"ogt", 2>; -def FCmpPredicateOGE : I64EnumAttrCase<"oge", 3>; -def FCmpPredicateOLT : I64EnumAttrCase<"olt", 4>; -def FCmpPredicateOLE : I64EnumAttrCase<"ole", 5>; -def FCmpPredicateONE : I64EnumAttrCase<"one", 6>; -def FCmpPredicateORD : I64EnumAttrCase<"ord", 7>; -def FCmpPredicateUEQ : I64EnumAttrCase<"ueq", 8>; -def FCmpPredicateUGT : I64EnumAttrCase<"ugt", 9>; -def FCmpPredicateUGE : I64EnumAttrCase<"uge", 10>; -def FCmpPredicateULT : I64EnumAttrCase<"ult", 11>; -def FCmpPredicateULE : I64EnumAttrCase<"ule", 12>; -def FCmpPredicateUNE : I64EnumAttrCase<"une", 13>; -def FCmpPredicateUNO : I64EnumAttrCase<"uno", 14>; -def FCmpPredicateTRUE : I64EnumAttrCase<"_true", 15>; - -def FCmpPredicate : I64EnumAttr< +// Predicates for float comparisons +def FCmpPredicateFALSE : LLVM_EnumAttrCase<"_false", "_false", "FCMP_FALSE", 0>; +def FCmpPredicateOEQ : LLVM_EnumAttrCase<"oeq", "oeq", "FCMP_OEQ", 1>; +def FCmpPredicateOGT : LLVM_EnumAttrCase<"ogt", "ogt", "FCMP_OGT", 2>; +def FCmpPredicateOGE : LLVM_EnumAttrCase<"oge", "oge", "FCMP_OGE", 3>; +def FCmpPredicateOLT : LLVM_EnumAttrCase<"olt", "olt", "FCMP_OLT", 4>; +def FCmpPredicateOLE : LLVM_EnumAttrCase<"ole", "ole", "FCMP_OLE", 5>; +def FCmpPredicateONE : LLVM_EnumAttrCase<"one", "one", "FCMP_ONE", 6>; +def FCmpPredicateORD : LLVM_EnumAttrCase<"ord", "ord", "FCMP_ORD", 7>; +def FCmpPredicateUEQ : LLVM_EnumAttrCase<"ueq", "ueq", "FCMP_UEQ", 8>; +def FCmpPredicateUGT : LLVM_EnumAttrCase<"ugt", "ugt", "FCMP_UGT", 9>; +def FCmpPredicateUGE : LLVM_EnumAttrCase<"uge", "uge", "FCMP_UGE", 10>; +def FCmpPredicateULT : LLVM_EnumAttrCase<"ult", "ult", "FCMP_ULT", 11>; +def FCmpPredicateULE : LLVM_EnumAttrCase<"ule", "ule", "FCMP_ULE", 12>; +def FCmpPredicateUNE : LLVM_EnumAttrCase<"une", "une", "FCMP_UNE", 13>; +def FCmpPredicateUNO : LLVM_EnumAttrCase<"uno", "uno", "FCMP_UNO", 14>; +def FCmpPredicateTRUE : LLVM_EnumAttrCase<"_true", "_true", "FCMP_TRUE", 15>; + +// A sentinel value that has no MLIR counterpart. +def ICmpPredicateBad : LLVM_EnumAttrCase<"", "", "BAD_ICMP_PREDICATE", 0>; + +// Predicates for integer comparisons. +def ICmpPredicateEQ : LLVM_EnumAttrCase<"eq", "eq", "ICMP_EQ", 0>; +def ICmpPredicateNE : LLVM_EnumAttrCase<"ne", "ne", "ICMP_NE", 1>; +def ICmpPredicateSLT : LLVM_EnumAttrCase<"slt", "slt", "ICMP_SLT", 2>; +def ICmpPredicateSLE : LLVM_EnumAttrCase<"sle", "sle", "ICMP_SLE", 3>; +def ICmpPredicateSGT : LLVM_EnumAttrCase<"sgt", "sgt", "ICMP_SGT", 4>; +def ICmpPredicateSGE : LLVM_EnumAttrCase<"sge", "sge", "ICMP_SGE", 5>; +def ICmpPredicateULT : LLVM_EnumAttrCase<"ult", "ult", "ICMP_ULT", 6>; +def ICmpPredicateULE : LLVM_EnumAttrCase<"ule", "ule", "ICMP_ULE", 7>; +def ICmpPredicateUGT : LLVM_EnumAttrCase<"ugt", "ugt", "ICMP_UGT", 8>; +def ICmpPredicateUGE : LLVM_EnumAttrCase<"uge", "uge", "ICMP_UGE", 9>; + +// A sentinel value that has no MLIR counterpart. +def FCmpPredicateBad : LLVM_EnumAttrCase<"", "", "BAD_FCMP_PREDICATE", 0>; + +// LLVM's predicate enum contains the floating-point and integer comparison +// cases, while the LLVM dialect uses two separate enums. The floating-point +// predicate enum thus defines all integer predicates as unsupported and +// vice versa. +def FCmpPredicate : LLVM_EnumAttr< "FCmpPredicate", + "::llvm::CmpInst::Predicate", "llvm.fcmp comparison predicate", [FCmpPredicateFALSE, FCmpPredicateOEQ, FCmpPredicateOGT, FCmpPredicateOGE, FCmpPredicateOLT, FCmpPredicateOLE, FCmpPredicateONE, FCmpPredicateORD, FCmpPredicateUEQ, FCmpPredicateUGT, FCmpPredicateUGE, FCmpPredicateULT, - FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE + FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE], + [ICmpPredicateEQ, ICmpPredicateNE, ICmpPredicateSLT, ICmpPredicateSLE, + ICmpPredicateSGT, ICmpPredicateSGE, ICmpPredicateULT, ICmpPredicateULE, + ICmpPredicateUGT, ICmpPredicateUGE, FCmpPredicateBad, ICmpPredicateBad ]> { let cppNamespace = "::mlir::LLVM"; } -//===----------------------------------------------------------------------===// -// ICmp Predicates -//===----------------------------------------------------------------------===// - -// Predicate for integer comparisons. -def ICmpPredicateEQ : I64EnumAttrCase<"eq", 0>; -def ICmpPredicateNE : I64EnumAttrCase<"ne", 1>; -def ICmpPredicateSLT : I64EnumAttrCase<"slt", 2>; -def ICmpPredicateSLE : I64EnumAttrCase<"sle", 3>; -def ICmpPredicateSGT : I64EnumAttrCase<"sgt", 4>; -def ICmpPredicateSGE : I64EnumAttrCase<"sge", 5>; -def ICmpPredicateULT : I64EnumAttrCase<"ult", 6>; -def ICmpPredicateULE : I64EnumAttrCase<"ule", 7>; -def ICmpPredicateUGT : I64EnumAttrCase<"ugt", 8>; -def ICmpPredicateUGE : I64EnumAttrCase<"uge", 9>; -def ICmpPredicate : I64EnumAttr< +def ICmpPredicate : LLVM_EnumAttr< "ICmpPredicate", - "llvm.icmp comparison predicate", + "::llvm::CmpInst::Predicate", + "lvm.icmp comparison predicate", [ICmpPredicateEQ, ICmpPredicateNE, ICmpPredicateSLT, ICmpPredicateSLE, ICmpPredicateSGT, ICmpPredicateSGE, ICmpPredicateULT, ICmpPredicateULE, - ICmpPredicateUGT, ICmpPredicateUGE]> { + ICmpPredicateUGT, ICmpPredicateUGE], + [FCmpPredicateFALSE, FCmpPredicateOEQ, FCmpPredicateOGT, FCmpPredicateOGE, + FCmpPredicateOLT, FCmpPredicateOLE, FCmpPredicateONE, FCmpPredicateORD, + FCmpPredicateUEQ, FCmpPredicateUGT, FCmpPredicateUGE, FCmpPredicateULT, + FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE, + FCmpPredicateBad, ICmpPredicateBad + ]> { let cppNamespace = "::mlir::LLVM"; } @@ -489,25 +529,6 @@ def Linkage : DialectAttr< "::mlir::LLVM::LinkageAttr::get($_builder.getContext(), $0)"; } -//===----------------------------------------------------------------------===// -// LoopOptions -//===----------------------------------------------------------------------===// - -def LOptDisableUnroll : I32EnumAttrCase<"disable_unroll", 1>; -def LOptDisableLICM : I32EnumAttrCase<"disable_licm", 2>; -def LOptInterleaveCount : I32EnumAttrCase<"interleave_count", 3>; -def LOptDisablePipeline : I32EnumAttrCase<"disable_pipeline", 4>; -def LOptPipelineInitiationInterval : I32EnumAttrCase<"pipeline_initiation_interval", 5>; - -def LoopOptionCase : I32EnumAttr< - "LoopOptionCase", - "LLVM loop option", - [LOptDisableUnroll, LOptDisableLICM, LOptInterleaveCount, - LOptDisablePipeline, LOptPipelineInitiationInterval - ]> { - let cppNamespace = "::mlir::LLVM"; -} - //===----------------------------------------------------------------------===// // UnnamedAddr //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index c1b82b91c110..dbb565dcd1c9 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -58,19 +58,31 @@ class LLVM_BinarySameArgsIntrOpF traits = []> : let arguments = !con(commonArgs, fmfArg); } -class LLVM_TernarySameArgsIntrOpF traits = []> : +class LLVM_TernarySameArgsIntrOpBase traits = [], bit requiresFastmath = 0> : LLVM_OneResultIntrOp { - let arguments = (ins LLVM_ScalarOrVectorOf:$a, - LLVM_ScalarOrVectorOf:$b, - LLVM_ScalarOrVectorOf:$c, - DefaultValuedAttr:$fastmathFlags); + requiresFastmath> { + dag commonArgs = (ins LLVM_ScalarOrVectorOf:$a, + LLVM_ScalarOrVectorOf:$b, + LLVM_ScalarOrVectorOf:$c); let assemblyFormat = "`(` operands `)` custom(attr-dict) `:` " "functional-type(operands, results)"; } +class LLVM_TernarySameArgsIntrOpI traits = []> : + LLVM_TernarySameArgsIntrOpBase { + let arguments = commonArgs; +} + +class LLVM_TernarySameArgsIntrOpF traits = []> : + LLVM_TernarySameArgsIntrOpBase { + dag fmfArg = ( + ins DefaultValuedAttr:$fastmathFlags); + let arguments = !con(commonArgs, fmfArg); +} + class LLVM_CountZerosIntrOp traits = []> : LLVM_OneResultIntrOp { @@ -109,8 +121,7 @@ def LLVM_FTruncOp : LLVM_UnaryIntrOpF<"trunc">; def LLVM_SqrtOp : LLVM_UnaryIntrOpF<"sqrt">; def LLVM_PowOp : LLVM_BinarySameArgsIntrOpF<"pow">; def LLVM_PowIOp : LLVM_OneResultIntrOp<"powi", [], [0,1], - [DeclareOpInterfaceMethods, Pure], - /*requiresFastmath=*/1> { + [Pure], /*requiresFastmath=*/1> { let arguments = (ins LLVM_ScalarOrVectorOf:$val, AnySignlessInteger:$power, @@ -119,9 +130,12 @@ def LLVM_PowIOp : LLVM_OneResultIntrOp<"powi", [], [0,1], "functional-type(operands, results)"; } def LLVM_BitReverseOp : LLVM_UnaryIntrOpI<"bitreverse">; +def LLVM_ByteSwapOp : LLVM_UnaryIntrOpI<"bswap">; def LLVM_CountLeadingZerosOp : LLVM_CountZerosIntrOp<"ctlz">; def LLVM_CountTrailingZerosOp : LLVM_CountZerosIntrOp<"cttz">; def LLVM_CtPopOp : LLVM_UnaryIntrOpI<"ctpop">; +def LLVM_FshlOp : LLVM_TernarySameArgsIntrOpI<"fshl">; +def LLVM_FshrOp : LLVM_TernarySameArgsIntrOpI<"fshr">; def LLVM_MaxNumOp : LLVM_BinarySameArgsIntrOpF<"maxnum">; def LLVM_MinNumOp : LLVM_BinarySameArgsIntrOpF<"minnum">; def LLVM_MaximumOp : LLVM_BinarySameArgsIntrOpF<"maximum">; @@ -262,6 +276,17 @@ def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> { let assemblyFormat = "$handle attr-dict"; } +// +// Saturation Arithmetic Intrinsics. +// + +def LLVM_SAddSat : LLVM_BinarySameArgsIntrOpI<"sadd.sat">; +def LLVM_UAddSat : LLVM_BinarySameArgsIntrOpI<"uadd.sat">; +def LLVM_SSubSat : LLVM_BinarySameArgsIntrOpI<"ssub.sat">; +def LLVM_USubSat : LLVM_BinarySameArgsIntrOpI<"usub.sat">; +def LLVM_SSHLSat : LLVM_BinarySameArgsIntrOpI<"sshl.sat">; +def LLVM_USHLSat : LLVM_BinarySameArgsIntrOpI<"ushl.sat">; + // // Debug function intrinsics. // @@ -291,9 +316,13 @@ class LLVM_DbgIntrOp : LLVM_IntrOp argOperand = moduleImport.convertValue(llvmOperands[0]); + + FailureOr argOperand = moduleImport.convertMetadataValue(llvmOperands[0]); + // Drop the intrinsic when its operand could not be converted. This can + // happen for use before definition cases that are allowed for debug + // intrinsics. if (failed(argOperand)) - return failure(); + return success(); $_op = $_builder.create<$_qualCppClassName>($_location, *argOperand, $_var_attr($varInfo)); }]; @@ -646,6 +675,26 @@ def LLVM_masked_compressstore let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf); } +// +// Trap intrinsics. +// + +def LLVM_Trap : LLVM_ZeroResultIntrOp<"trap">; + +def LLVM_DebugTrap : LLVM_ZeroResultIntrOp<"debugtrap">; + +def LLVM_UBSanTrap : LLVM_ZeroResultIntrOp<"ubsantrap"> { + let arguments = (ins I8Attr:$failureKind); + string llvmBuilder = [{ + builder.CreateIntrinsic( + llvm::Intrinsic::ubsantrap, {}, {builder.getInt8($failureKind)}); + }]; + string mlirBuilder = [{ + $_op = + $_builder.create($_location, $_int_attr($failureKind)); + }]; +} + /// Create a call to vscale intrinsic. def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>; diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index eda30e36086f..754656f3dec2 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -36,28 +36,39 @@ def LLVM_Dialect : Dialect { let extraClassDeclaration = [{ /// Name of the data layout attributes. static StringRef getDataLayoutAttrName() { return "llvm.data_layout"; } - static StringRef getAlignAttrName() { return "llvm.align"; } - static StringRef getNoAliasAttrName() { return "llvm.noalias"; } - static StringRef getReadonlyAttrName() { return "llvm.readonly"; } static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; } static StringRef getAliasScopesAttrName() { return "alias_scopes"; } - static StringRef getLoopAttrName() { return "llvm.loop"; } - static StringRef getParallelAccessAttrName() { return "parallel_access"; } - static StringRef getLoopOptionsAttrName() { return "options"; } static StringRef getAccessGroupsAttrName() { return "access_groups"; } - static StringRef getStructAttrsAttrName() { return "llvm.struct_attrs"; } + + /// Names of llvm parameter attributes. + static StringRef getAlignAttrName() { return "llvm.align"; } + static StringRef getAllocAlignAttrName() { return "llvm.allocalign"; } + static StringRef getAllocatedPointerAttrName() { return "llvm.allocptr"; } static StringRef getByValAttrName() { return "llvm.byval"; } static StringRef getByRefAttrName() { return "llvm.byref"; } - static StringRef getStructRetAttrName() { return "llvm.sret"; } - static StringRef getInAllocaAttrName() { return "llvm.inalloca"; } static StringRef getNoUndefAttrName() { return "llvm.noundef"; } + static StringRef getDereferenceableAttrName() { return "llvm.dereferenceable"; } + static StringRef getDereferenceableOrNullAttrName() { return "llvm.dereferenceable_or_null"; } + static StringRef getInAllocaAttrName() { return "llvm.inalloca"; } + static StringRef getInRegAttrName() { return "llvm.inreg"; } + static StringRef getNestAttrName() { return "llvm.nest"; } + static StringRef getNoAliasAttrName() { return "llvm.noalias"; } + static StringRef getNoCaptureAttrName() { return "llvm.nocapture"; } + static StringRef getNoFreeAttrName() { return "llvm.nofree"; } + static StringRef getNonNullAttrName() { return "llvm.nonnull"; } + static StringRef getPreallocatedAttrName() { return "llvm.preallocated"; } + static StringRef getReadonlyAttrName() { return "llvm.readonly"; } + static StringRef getReturnedAttrName() { return "llvm.returned"; } static StringRef getSExtAttrName() { return "llvm.signext"; } + static StringRef getStackAlignmentAttrName() { return "llvm.alignstack"; } + static StringRef getStructRetAttrName() { return "llvm.sret"; } + static StringRef getWriteOnlyAttrName() { return "llvm.writeonly"; } static StringRef getZExtAttrName() { return "llvm.zeroext"; } - static StringRef getTBAAAttrName() { return "llvm.tbaa"; } - - /// Verifies if the attribute is a well-formed value for "llvm.struct_attrs" - static LogicalResult verifyStructAttr( - Operation *op, Attribute attr, Type annotatedType); + // TODO Restrict the usage of this to parameter attributes once there is an + // alternative way of modeling memory effects on FunctionOpInterface. + /// Name of the attribute that will cause the creation of a readnone memory + /// effect when lowering to the LLVMDialect. + static StringRef getReadnoneAttrName() { return "llvm.readnone"; } /// Verifies if the given string is a well-formed data layout descriptor. /// Uses `reportError` to report errors. @@ -75,18 +86,17 @@ def LLVM_Dialect : Dialect { /// Returns `true` if the given type is compatible with the LLVM dialect. static bool isCompatibleType(Type); - /// TODO Remove this once there is an alternative way of modeling memory - /// effects on FunctionOpInterface. - /// Name of the attribute that will cause the creation of a readnone memory - /// effect when lowering to the LLVMDialect. - static StringRef getReadnoneAttrName() { - return "llvm.readnone"; - } Type parseType(DialectAsmParser &p) const override; void printType(Type, DialectAsmPrinter &p) const override; private: + /// Verifies a parameter attribute attached to a parameter of type + /// paramType. + LogicalResult verifyParameterAttribute(Operation *op, + Type paramType, + NamedAttribute paramAttr); + /// Register all types. void registerTypes(); @@ -293,14 +303,24 @@ class LLVM_EnumAttrCase : // - `name`: name of the C++ enum class in MLIR API; // - `llvmName`: name of the C++ enum in LLVM API; // - `description`: textual description for documentation purposes; -// - `cases`: list of enum cases. +// - `cases`: list of enum cases; +// - `unsupportedCases`: optional list of unsupported enum cases. // For example, `LLVM_EnumAttr cases> : + list cases, + list unsupportedCases = []> : I64EnumAttr { + + // List of unsupported cases that have no conversion to an MLIR value. + list unsupported = unsupportedCases; + // The equivalent enum class name in LLVM. string llvmClassName = llvmName; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index c31245cf6f03..bfacb18f3a0e 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -122,12 +122,13 @@ def LLVM_ICmpOp : LLVM_ArithmeticCmpOp<"icmp", [Pure]> { let hasCustomAssemblyFormat = 1; string llvmInstName = "ICmp"; string llvmBuilder = [{ - $res = builder.CreateICmp(getLLVMCmpPredicate($predicate), $lhs, $rhs); + $res = builder.CreateICmp( + convertICmpPredicateToLLVM($predicate), $lhs, $rhs); }]; string mlirBuilder = [{ auto *iCmpInst = cast(inst); - $res = $_builder.create<$_qualCppClassName>( - $_location, getICmpPredicate(iCmpInst->getPredicate()), $lhs, $rhs); + $res = $_builder.create<$_qualCppClassName>($_location, + convertICmpPredicateFromLLVM(iCmpInst->getPredicate()), $lhs, $rhs); }]; // Set the $predicate index to -1 to indicate there is no matching operand // and decrement the following indices. @@ -145,12 +146,12 @@ def LLVM_FCmpOp : LLVM_ArithmeticCmpOp<"fcmp", [ let hasCustomAssemblyFormat = 1; string llvmInstName = "FCmp"; string llvmBuilder = [{ - $res = builder.CreateFCmp(getLLVMCmpPredicate($predicate), $lhs, $rhs); + $res = builder.CreateFCmp(convertFCmpPredicateToLLVM($predicate), $lhs, $rhs); }]; string mlirBuilder = [{ auto *fCmpInst = cast(inst); auto op = $_builder.create<$_qualCppClassName>( - $_location, getFCmpPredicate(fCmpInst->getPredicate()), $lhs, $rhs); + $_location, convertFCmpPredicateFromLLVM(fCmpInst->getPredicate()), $lhs, $rhs); moduleImport.setFastmathFlagsAttr(inst, op); $res = op; }]; @@ -168,9 +169,8 @@ def LLVM_FRemOp : LLVM_FloatArithmeticOp<"frem", "FRem">; def LLVM_FNegOp : LLVM_UnaryFloatArithmeticOp< LLVM_ScalarOrVectorOf, "fneg", "FNeg">; -// Common code definition that is used to verify and set the alignment attribute -// of LLVM ops that accept such an attribute. -class MemoryOpWithAlignmentBase { +// Common code definitions used to set memory operation attributes and flags. +class MemoryOpBase { code setAlignmentCode = [{ if ($alignment.has_value()) { auto align = *$alignment; @@ -178,35 +178,39 @@ class MemoryOpWithAlignmentBase { inst->setAlignment(llvm::Align(align)); } }]; -} - -// Code definition that is used for nontemporal metadata creation. -class MemoryOpWithAlignmentAndAttributes : MemoryOpWithAlignmentBase { + code setVolatileCode = [{ + inst->setVolatile($volatile_); + }]; + code setSyncScopeCode = [{ + if ($syncscope.has_value()) { + llvm::LLVMContext &llvmContext = builder.getContext(); + inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope)); + } + }]; + code setOrderingCode = [{ + inst->setAtomic(convertAtomicOrderingToLLVM($ordering)); + }]; code setNonTemporalMetadataCode = [{ if ($nontemporal) { - llvm::Module *module = builder.GetInsertBlock()->getModule(); llvm::MDNode *metadata = llvm::MDNode::get( inst->getContext(), llvm::ConstantAsMetadata::get( builder.getInt32(1))); - inst->setMetadata(module->getMDKindID("nontemporal"), metadata); + inst->setMetadata(llvm::LLVMContext::MD_nontemporal, metadata); } }]; - code setAccessGroupsMetadataCode = [{ moduleTranslation.setAccessGroupsMetadata(op, inst); }]; - code setAliasScopeMetadataCode = [{ moduleTranslation.setAliasScopeMetadata(op, inst); }]; - code setTBAAMetadataCode = [{ moduleTranslation.setTBAAMetadata(op, inst); }]; } // Memory-related operations. -def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpWithAlignmentBase { +def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpBase { let arguments = (ins AnyInteger:$arraySize, OptionalAttr:$alignment, OptionalAttr:$elem_type); @@ -343,18 +347,54 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure]> { let hasVerifier = 1; } -def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpWithAlignmentAndAttributes { +def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { let arguments = (ins Arg, "", [MemRead]>:$addr, OptionalAttr:$access_groups, OptionalAttr:$alias_scopes, OptionalAttr:$noalias_scopes, - OptionalAttr:$alignment, UnitAttr:$volatile_, - UnitAttr:$nontemporal); + OptionalAttr:$tbaa, + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr:$ordering, + OptionalAttr:$syncscope); let results = (outs LLVM_LoadableType:$res); string llvmInstName = "Load"; + let description = [{ + The `load` operation is used to read from memory. A load may be marked as + atomic, volatile, and/or nontemporal, and takes a number of optional + attributes that specify aliasing information. + + An atomic load only supports a limited set of pointer, integer, and + floating point types, and requires an explicit alignment. + + Examples: + ```mlir + // A volatile load of a float variable. + %0 = llvm.load volatile %ptr : !llvm.ptr -> f32 + + // A nontemporal load of a float variable. + %0 = llvm.load %ptr {nontemporal} : !llvm.ptr -> f32 + + // An atomic load of an integer variable. + %0 = llvm.load %ptr atomic monotonic {alignment = 8 : i64} + : !llvm.ptr -> i64 + ``` + + See the following link for more details: + https://llvm.org/docs/LangRef.html#load-instruction + }]; + let assemblyFormat = [{ + (`volatile` $volatile_^)? $addr + (`atomic` (`syncscope` `(` $syncscope^ `)`)? $ordering^)? + attr-dict `:` custom(type($addr), type($res)) + }]; string llvmBuilder = [{ auto *inst = builder.CreateLoad($_resultType, $addr, $volatile_); - }] # setAlignmentCode + }] # setOrderingCode + # setSyncScopeCode + # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode # setAliasScopeMetadataCode @@ -362,52 +402,95 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpWithAlignmentAndAttributes { # [{ $res = inst; }]; - // FIXME: Import attributes. string mlirBuilder = [{ - $res = $_builder.create($_location, $_resultType, $addr); + auto *loadInst = cast(inst); + unsigned alignment = loadInst->getAlign().value(); + $res = $_builder.create($_location, $_resultType, $addr, + alignment, loadInst->isVolatile(), + loadInst->hasMetadata(llvm::LLVMContext::MD_nontemporal), + convertAtomicOrderingFromLLVM(loadInst->getOrdering()), + getLLVMSyncScope(loadInst)); }]; let builders = [ OpBuilder<(ins "Value":$addr, CArg<"unsigned", "0">:$alignment, - CArg<"bool", "false">:$isVolatile, CArg<"bool", "false">:$isNonTemporal), - [{ - auto type = addr.getType().cast().getElementType(); - assert(type && "must provide explicit element type to the constructor " - "when the pointer type is opaque"); - build($_builder, $_state, type, addr, alignment, isVolatile, isNonTemporal); - }]>, - OpBuilder<(ins "Type":$t, "Value":$addr, + CArg<"bool", "false">:$isVolatile, CArg<"bool", "false">:$isNonTemporal)>, + OpBuilder<(ins "Type":$type, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, - CArg<"bool", "false">:$isNonTemporal)>,]; - let hasCustomAssemblyFormat = 1; + CArg<"bool", "false">:$isNonTemporal, + CArg<"AtomicOrdering", "AtomicOrdering::not_atomic">:$ordering, + CArg<"StringRef", "StringRef()">:$syncscope)> + ]; let hasVerifier = 1; } -def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpWithAlignmentAndAttributes { +def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { let arguments = (ins LLVM_LoadableType:$value, Arg,"",[MemWrite]>:$addr, OptionalAttr:$access_groups, OptionalAttr:$alias_scopes, OptionalAttr:$noalias_scopes, - OptionalAttr:$alignment, UnitAttr:$volatile_, - UnitAttr:$nontemporal); + OptionalAttr:$tbaa, + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr:$ordering, + OptionalAttr:$syncscope); string llvmInstName = "Store"; + let description = [{ + The `store` operation is used to write to memory. A store may be marked as + atomic, volatile, and/or nontemporal, and takes a number of optional + attributes that specify aliasing information. + + An atomic store only supports a limited set of pointer, integer, and + floating point types, and requires an explicit alignment. + + Examples: + ```mlir + // A volatile store of a float variable. + llvm.store volatile %val, %ptr : f32, !llvm.ptr + + // A nontemporal store of a float variable. + llvm.store %val, %ptr {nontemporal} : f32, !llvm.ptr + + // An atomic store of an integer variable. + llvm.store %val, %ptr atomic monotonic {alignment = 8 : i64} + : i64, !llvm.ptr + ``` + + See the following link for more details: + https://llvm.org/docs/LangRef.html#store-instruction + }]; + let assemblyFormat = [{ + (`volatile` $volatile_^)? $value `,` $addr + (`atomic` (`syncscope` `(` $syncscope^ `)`)? $ordering^)? + attr-dict `:` custom(type($value), type($addr)) + }]; string llvmBuilder = [{ auto *inst = builder.CreateStore($value, $addr, $volatile_); - }] # setAlignmentCode + }] # setOrderingCode + # setSyncScopeCode + # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode # setAliasScopeMetadataCode # setTBAAMetadataCode; - // FIXME: Import attributes. string mlirBuilder = [{ - $_op = $_builder.create($_location, $value, $addr); + auto *storeInst = cast(inst); + unsigned alignment = storeInst->getAlign().value(); + $_op = $_builder.create($_location, $value, $addr, + alignment, storeInst->isVolatile(), + storeInst->hasMetadata(llvm::LLVMContext::MD_nontemporal), + convertAtomicOrderingFromLLVM(storeInst->getOrdering()), + getLLVMSyncScope(storeInst)); }]; let builders = [ OpBuilder<(ins "Value":$value, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, - CArg<"bool", "false">:$isNonTemporal)> - ]; - let hasCustomAssemblyFormat = 1; + CArg<"bool", "false">:$isNonTemporal, + CArg<"AtomicOrdering", "AtomicOrdering::not_atomic">:$ordering, + CArg<"StringRef", "StringRef()">:$syncscope)> + ]; let hasVerifier = 1; } @@ -429,6 +512,7 @@ class LLVM_CastOp { let hasFolder = 1; + let hasVerifier = 1; } def LLVM_AddrSpaceCastOp : LLVM_CastOp<"addrspacecast", "AddrSpaceCast", LLVM_ScalarOrVectorOf, @@ -513,6 +597,9 @@ def LLVM_LandingpadOp : LLVM_Op<"landingpad"> { // CallOp //===----------------------------------------------------------------------===// +// FIXME: Add a type attribute that carries the LLVM function type to support +// indirect calls to variadic functions. The type attribute is necessary to +// distinguish normal and variadic arguments. def LLVM_CallOp : LLVM_Op<"call", [DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, @@ -527,9 +614,10 @@ def LLVM_CallOp : LLVM_Op<"call", The `call` instruction supports both direct and indirect calls. Direct calls start with a function name (`@`-prefixed) and indirect calls start with an SSA value (`%`-prefixed). The direct callee, if present, is stored as a - function attribute `callee`. The trailing type of the instruction is always - the MLIR function type, which may be different from the indirect callee that - has the wrapped LLVM IR function type. + function attribute `callee`. The trailing type list contains the optional + indirect callee type and the MLIR function type, which differs from the + LLVM function type that uses a explicit void type to model functions that do + not return a value. Examples: @@ -541,7 +629,7 @@ def LLVM_CallOp : LLVM_Op<"call", llvm.call @bar(%0) : (f32) -> () // Indirect call with an argument and without a result. - llvm.call %1(%0) : (f32) -> () + llvm.call %1(%0) : !llvm.ptr, (f32) -> () ``` }]; @@ -756,7 +844,10 @@ def LLVM_FreezeOp : LLVM_Op<"freeze", [SameOperandsAndResultType]> { // Terminators. def LLVM_BrOp : LLVM_TerminatorOp<"br", [DeclareOpInterfaceMethods, Pure]> { - let arguments = (ins Variadic:$destOperands); + let arguments = (ins + Variadic:$destOperands, + OptionalAttr:$loop_annotation + ); let successors = (successor AnySuccessor:$dest); let assemblyFormat = [{ $dest (`(` $destOperands^ `:` type($destOperands) `)`)? attr-dict @@ -765,8 +856,12 @@ def LLVM_BrOp : LLVM_TerminatorOp<"br", OpBuilder<(ins "Block *":$dest), [{ build($_builder, $_state, ValueRange(), dest); }]>, + OpBuilder<(ins "ValueRange":$operands, "Block *":$dest), [{ + build($_builder, $_state, operands, /*loop_annotation=*/{}, dest); + }]>, LLVM_TerminatorPassthroughOpBuilder ]; + let hasVerifier = 1; } def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br", [AttrSizedOperandSegments, DeclareOpInterfaceMethods, @@ -774,7 +869,8 @@ def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br", let arguments = (ins I1:$condition, Variadic:$trueDestOperands, Variadic:$falseDestOperands, - OptionalAttr:$branch_weights); + OptionalAttr:$branch_weights, + OptionalAttr:$loop_annotation); let successors = (successor AnySuccessor:$trueDest, AnySuccessor:$falseDest); let assemblyFormat = [{ $condition ( `weights` `(` $branch_weights^ `)` )? `,` @@ -787,23 +883,20 @@ def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br", OpBuilder<(ins "Value":$condition, "Block *":$trueDest, "ValueRange":$trueOperands, "Block *":$falseDest, "ValueRange":$falseOperands, - CArg<"std::optional>", "{}">:$weights), - [{ - ElementsAttr weightsAttr; - if (weights) { - weightsAttr = - $_builder.getI32VectorAttr({static_cast(weights->first), - static_cast(weights->second)}); - } - build($_builder, $_state, condition, trueOperands, falseOperands, weightsAttr, - trueDest, falseDest); - }]>, + CArg<"std::optional>", "{}">:$weights)>, OpBuilder<(ins "Value":$condition, "Block *":$trueDest, "Block *":$falseDest, CArg<"ValueRange", "{}">:$falseOperands), [{ build($_builder, $_state, condition, trueDest, ValueRange(), falseDest, falseOperands); + }]>, + OpBuilder<(ins "Value":$condition, "ValueRange":$trueOperands, "ValueRange":$falseOperands, + "ElementsAttr":$branchWeights, "Block *":$trueDest, "Block *":$falseDest), + [{ + build($_builder, $_state, condition, trueOperands, falseOperands, branchWeights, + {}, trueDest, falseDest); }]>, LLVM_TerminatorPassthroughOpBuilder]; + let hasVerifier = 1; } //===----------------------------------------------------------------------===// @@ -1064,6 +1157,7 @@ def LLVM_AliasScopeMetadataOp : LLVM_Op<"alias_scope", [ https://llvm.org/docs/LangRef.html#noalias-and-alias-scope-metadata }]; let assemblyFormat = "$sym_name attr-dict"; + let hasVerifier = 1; } def LLVM_AccessGroupMetadataOp : LLVM_Op<"access_group", [ @@ -1650,85 +1744,124 @@ def LLVM_ConstantOp def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, AnyInteger]>; -// FIXME: Need to add alignment and syncscope attribute to MLIR atomicrmw -// operation. -def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw"> { +def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ + TypesMatchWith<"result #0 and operand #1 have the same type", + "val", "res", "$_self">]>, MemoryOpBase { let arguments = (ins AtomicBinOp:$bin_op, LLVM_PointerTo:$ptr, - LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering); + LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$volatile_); let results = (outs LLVM_AtomicRMWType:$res); - let hasCustomAssemblyFormat = 1; - let hasVerifier = 1; + let assemblyFormat = [{ + (`volatile` $volatile_^)? $bin_op $ptr `,` $val + (`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict `:` + qualified(type($ptr)) `,` type($val) + }]; string llvmInstName = "AtomicRMW"; string llvmBuilder = [{ - $res = builder.CreateAtomicRMW(getLLVMAtomicBinOp($bin_op), $ptr, $val, - llvm::MaybeAlign(), - getLLVMAtomicOrdering($ordering)); - }]; + auto *inst = builder.CreateAtomicRMW( + convertAtomicBinOpToLLVM($bin_op), $ptr, $val, llvm::MaybeAlign(), + convertAtomicOrderingToLLVM($ordering)); + $res = inst; + }] # setVolatileCode + # setSyncScopeCode + # setAlignmentCode; string mlirBuilder = [{ auto *atomicInst = cast(inst); - $res = $_builder.create($_location, $_resultType, - getLLVMAtomicBinOp(atomicInst->getOperation()), $ptr, $val, - getLLVMAtomicOrdering(atomicInst->getOrdering())); + unsigned alignment = atomicInst->getAlign().value(); + $res = $_builder.create($_location, + convertAtomicBinOpFromLLVM(atomicInst->getOperation()), $ptr, $val, + convertAtomicOrderingFromLLVM(atomicInst->getOrdering()), + getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile()); }]; - // Only $ptr and $val are llvm instruction operands. - list llvmArgIndices = [-1, 0, 1, -1]; + list llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1]; + let builders = [ + OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val, + "LLVM::AtomicOrdering":$ordering, + CArg<"StringRef", "StringRef()">:$syncscope, + CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile + )> + ]; + let hasVerifier = 1; } def LLVM_AtomicCmpXchgType : AnyTypeOf<[AnyInteger, LLVM_AnyPointer]>; -def LLVM_AtomicCmpXchgResultType : Type().getBody().size() == 2">, - SubstLeaves<"$_self", - "$_self.cast<::mlir::LLVM::LLVMStructType>().getBody()[0]", - LLVM_AtomicCmpXchgType.predicate>, - SubstLeaves<"$_self", - "$_self.cast<::mlir::LLVM::LLVMStructType>().getBody()[1]", - I1.predicate>]>, - "an LLVM struct type with any integer or pointer followed by a single-bit " - "integer">; - -// FIXME: Need to add alignment attribute to MLIR cmpxchg operation. -def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg"> { + +def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ + TypesMatchWith<"operand #1 and operand #2 have the same type", + "val", "cmp", "$_self">, + TypesMatchWith<"result #0 has an LLVM struct type consisting of " + "the type of operand #2 and a bool", "val", "res", + "getValAndBoolStructType($_self)">]>, MemoryOpBase { let arguments = (ins LLVM_PointerTo:$ptr, LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, AtomicOrdering:$success_ordering, - AtomicOrdering:$failure_ordering); - let results = (outs LLVM_AtomicCmpXchgResultType:$res); - let hasCustomAssemblyFormat = 1; - let hasVerifier = 1; + AtomicOrdering:$failure_ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$weak, + UnitAttr:$volatile_); + let results = (outs LLVM_AnyStruct:$res); + let assemblyFormat = [{ + (`weak` $weak^)? (`volatile` $volatile_^)? $ptr `,` $cmp `,` $val + (`syncscope` `(` $syncscope^ `)`)? $success_ordering $failure_ordering + attr-dict `:` qualified(type($ptr)) `,` type($val) + }]; string llvmInstName = "AtomicCmpXchg"; string llvmBuilder = [{ - $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(), - getLLVMAtomicOrdering($success_ordering), - getLLVMAtomicOrdering($failure_ordering)); - }]; + auto *inst = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, + llvm::MaybeAlign(), convertAtomicOrderingToLLVM($success_ordering), + convertAtomicOrderingToLLVM($failure_ordering)); + $res = inst; + inst->setWeak($weak); + }] # setVolatileCode + # setSyncScopeCode + # setAlignmentCode; string mlirBuilder = [{ auto *cmpXchgInst = cast(inst); + unsigned alignment = cmpXchgInst->getAlign().value(); $res = $_builder.create( - $_location, $_resultType, $ptr, $cmp, $val, - getLLVMAtomicOrdering(cmpXchgInst->getSuccessOrdering()), - getLLVMAtomicOrdering(cmpXchgInst->getFailureOrdering())); + $_location, $ptr, $cmp, $val, + convertAtomicOrderingFromLLVM(cmpXchgInst->getSuccessOrdering()), + convertAtomicOrderingFromLLVM(cmpXchgInst->getFailureOrdering()), + getLLVMSyncScope(cmpXchgInst), alignment, cmpXchgInst->isWeak(), + cmpXchgInst->isVolatile()); }]; + let builders = [ + OpBuilder<(ins "Value":$ptr, "Value":$cmp, "Value":$val, + "LLVM::AtomicOrdering":$successOrdering, + "LLVM::AtomicOrdering":$failureOrdering, + CArg<"StringRef", "StringRef()">:$syncscope, + CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isWeak, + CArg<"bool", "false">:$isVolatile + )> + ]; + let hasVerifier = 1; } -def LLVM_FenceOp : LLVM_Op<"fence"> { - let arguments = (ins AtomicOrdering:$ordering, StrAttr:$syncscope); - let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; +def LLVM_FenceOp : LLVM_Op<"fence">, MemoryOpBase { + let arguments = (ins AtomicOrdering:$ordering, + OptionalAttr:$syncscope); + let assemblyFormat = "(`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict"; string llvmInstName = "Fence"; let llvmBuilder = [{ - llvm::LLVMContext &llvmContext = builder.getContext(); - builder.CreateFence(getLLVMAtomicOrdering($ordering), - llvmContext.getOrInsertSyncScopeID($syncscope)); - }]; + auto *inst = builder.CreateFence(convertAtomicOrderingToLLVM($ordering)); + }] # setSyncScopeCode; string mlirBuilder = [{ llvm::FenceInst *fenceInst = cast(inst); $_op = $_builder.create( $_location, - getLLVMAtomicOrdering(fenceInst->getOrdering()), + convertAtomicOrderingFromLLVM(fenceInst->getOrdering()), getLLVMSyncScope(fenceInst)); }]; - let hasCustomAssemblyFormat = 1; + let builders = [ + LLVM_VoidResultTypeOpBuilder, + LLVM_ZeroResultOpBuilder, + OpBuilder<(ins "LLVM::AtomicOrdering":$ordering, + CArg<"StringRef", "StringRef()">:$syncscope)> + ]; let hasVerifier = 1; } diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td index c2c0d0afe4f1..8fe96d16d5ec 100644 --- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td +++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td @@ -1155,7 +1155,8 @@ def LoadOp : MemRef_Op<"load", let arguments = (ins Arg:$memref, - Variadic:$indices); + Variadic:$indices, + DefaultValuedOptionalAttr:$nontemporal); let results = (outs AnyType:$result); let extraClassDeclaration = [{ @@ -1690,7 +1691,8 @@ def MemRef_StoreOp : MemRef_Op<"store", let arguments = (ins AnyType:$value, Arg:$memref, - Variadic:$indices); + Variadic:$indices, + DefaultValuedOptionalAttr:$nontemporal); let builders = [ OpBuilder<(ins "Value":$valueToStore, "Value":$memref), [{ diff --git a/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h b/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h index 4059cd4cc401..d3a16de8c551 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h +++ b/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h @@ -118,14 +118,14 @@ struct SparseCompilerOptions } /// Projects out the options for `createConvertVectorToLLVMPass`. - LowerVectorToLLVMOptions lowerVectorToLLVMOptions() const { - LowerVectorToLLVMOptions opts{}; - opts.enableReassociateFPReductions(reassociateFPReductions); - opts.enableIndexOptimizations(force32BitVectorIndices); - opts.enableArmNeon(armNeon); - opts.enableArmSVE(armSVE); - opts.enableAMX(amx); - opts.enableX86Vector(x86Vector); + ConvertVectorToLLVMPassOptions lowerVectorToLLVMOptions() const { + ConvertVectorToLLVMPassOptions opts{}; + opts.reassociateFPReductions = reassociateFPReductions; + opts.force32BitVectorIndices = force32BitVectorIndices; + opts.armNeon = armNeon; + opts.armSVE = armSVE; + opts.amx = amx; + opts.x86Vector = x86Vector; return opts; } }; diff --git a/mlir/include/mlir/IR/Dominance.h b/mlir/include/mlir/IR/Dominance.h index 30f5674ad71d..49216b80284a 100644 --- a/mlir/include/mlir/IR/Dominance.h +++ b/mlir/include/mlir/IR/Dominance.h @@ -22,9 +22,8 @@ #include "mlir/IR/RegionGraphTraits.h" #include "llvm/Support/GenericDomTree.h" -extern template class llvm::DominatorTreeBase; -extern template class llvm::DominatorTreeBase; -extern template class llvm::DomTreeNodeBase; +extern template class llvm::DominatorTreeOnView; +extern template class llvm::DominatorTreeOnView; namespace mlir { using DominanceInfoNode = llvm::DomTreeNodeBase; diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h index 92f4c610a903..e73434a9f080 100644 --- a/mlir/include/mlir/IR/OpImplementation.h +++ b/mlir/include/mlir/IR/OpImplementation.h @@ -213,6 +213,8 @@ class AsmPrinter { printArrowTypeList(results); } + virtual const void *getCurrentTypeOrAttrAlias() const; + protected: /// Initialize the printer with no internal implementation. In this case, all /// virtual methods of this class must be overriden. diff --git a/mlir/include/mlir/Support/IndentedOstream.h b/mlir/include/mlir/Support/IndentedOstream.h index 3d19a1e1ede3..7b19d37c7f38 100644 --- a/mlir/include/mlir/Support/IndentedOstream.h +++ b/mlir/include/mlir/Support/IndentedOstream.h @@ -22,7 +22,7 @@ namespace mlir { /// raw_ostream subclass that simplifies indention a sequence of code. class raw_indented_ostream : public raw_ostream { public: - explicit raw_indented_ostream(llvm::raw_ostream &os) : os(os) { + explicit raw_indented_ostream(llvm::raw_ostream &os, int indentSize = 2) : indentSize(indentSize), os(os) { SetUnbuffered(); } @@ -92,7 +92,7 @@ class raw_indented_ostream : public raw_ostream { uint64_t current_pos() const override { return os.tell(); } /// Constant indent added/removed. - static constexpr int indentSize = 2; + int indentSize; /// Tracker for current indentation. int currentIndent = 0; diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index 62a60a4c0d46..be4f6e5717b1 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -33,6 +33,7 @@ namespace LLVM { namespace detail { class DebugImporter; +class LoopAnnotationImporter; } // namespace detail /// Module import implementation class that provides methods to import globals @@ -118,6 +119,11 @@ class ModuleImport { /// LLVM values. FailureOr convertValue(llvm::Value *value); + /// Converts an LLVM metadata value to an MLIR value, or returns failure if + /// the conversion fails. Uses the `convertConstant` method to translate + /// constant LLVM values. + FailureOr convertMetadataValue(llvm::Value *value); + /// Converts a range of LLVM values to a range of MLIR values using the /// `convertValue` method, or returns failure if the conversion fails. FailureOr> convertValues(ArrayRef values); @@ -149,17 +155,38 @@ class ModuleImport { /// implement the fastmath interface. void setFastmathFlagsAttr(llvm::Instruction *inst, Operation *op) const; - /// Converts LLVM metadata to corresponding MLIR representation, - /// e.g. metadata nodes referenced via !tbaa are converted to - /// TBAA operations hosted inside a MetadataOp. + /// Converts all LLVM metadata nodes that translate to operations nested in a + /// global metadata operation, such as alias analysis or access group + /// metadata, and builds a map from the metadata nodes to the symbols pointing + /// to the converted operations. Returns success if all conversions succeed + /// and failure otherwise. + // Note: All metadata is nested inside a single global metadata operation to + // minimize the number of symbols that pollute the global namespace. LogicalResult convertMetadata(); - /// Returns SymbolRefAttr representing TBAA metadata `node` - /// in `tbaaMapping`. - SymbolRefAttr lookupTBAAAttr(const llvm::MDNode *node) { + /// Returns the MLIR symbol reference mapped to the given LLVM TBAA + /// metadata `node`. + SymbolRefAttr lookupTBAAAttr(const llvm::MDNode *node) const { return tbaaMapping.lookup(node); } + /// Returns the symbol references pointing to the access group operations that + /// map to the access group nodes starting from the access group metadata + /// `node`. Returns failure, if any of the symbol references cannot be found. + FailureOr> + lookupAccessGroupAttrs(const llvm::MDNode *node) const; + + /// Returns the loop annotation attribute that corresponds to the given LLVM + /// loop metadata `node`. + LoopAnnotationAttr translateLoopAnnotationAttr(const llvm::MDNode *node, + Location loc) const; + + /// Returns the symbol references pointing to the alias scope operations that + /// map to the alias scope nodes starting from the metadata `node`. Returns + /// failure, if any of the symbol references cannot be found. + FailureOr> + lookupAliasScopeAttrs(const llvm::MDNode *node) const; + private: /// Clears the block and value mapping before processing a new region. void clearBlockAndValueMapping() { @@ -210,6 +237,14 @@ class ModuleImport { LogicalResult convertCallTypeAndOperands(llvm::CallBase *callInst, SmallVectorImpl &types, SmallVectorImpl &operands); + /// Converts the parameter attributes attached to `func` and adds them to the + /// `funcOp`. + void convertParameterAttributes(llvm::Function *func, LLVMFuncOp funcOp, + OpBuilder &builder); + /// Converts the AttributeSet of one parameter in LLVM IR to a corresponding + /// DictionaryAttr for the LLVM dialect. + DictionaryAttr convertParameterAttribute(llvm::AttributeSet llvmParamAttrs, + OpBuilder &builder); /// Returns the builtin type equivalent to be used in attributes for the given /// LLVM IR dialect type. Type getStdTypeForAttr(Type type); @@ -229,25 +264,27 @@ class ModuleImport { /// them fails. All operations are inserted at the start of the current /// function entry block. FailureOr convertConstantExpr(llvm::Constant *constant); - /// Returns symbol name to be used for MetadataOp containing - /// TBAA metadata operations. It must not conflict with the user - /// name space. - StringRef getTBAAMetadataOpName() const { return "__tbaa"; } - /// Returns a terminated MetadataOp into which TBAA metadata - /// operations can be placed. The MetadataOp is created - /// on the first invocation of this function. - MetadataOp getTBAAMetadataOp(); + /// Returns a global metadata operation that serves as a container for LLVM + /// metadata that converts to MLIR operations. Creates the global metadata + /// operation on the first invocation. + MetadataOp getGlobalMetadataOp(); /// Performs conversion of LLVM TBAA metadata starting from /// `node`. On exit from this function all nodes reachable /// from `node` are converted, and tbaaMapping map is updated /// (unless all dependencies have been converted by a previous /// invocation of this function). LogicalResult processTBAAMetadata(const llvm::MDNode *node); - /// Returns unique string name of a symbol that may be used - /// for a TBAA metadata operation. The name will contain - /// the provided `basename` and will be uniqued via - /// tbaaNodeCounter (see below). - std::string getNewTBAANodeName(StringRef basename); + /// Converts all LLVM access groups starting from `node` to MLIR access group + /// operations and stores a mapping from every nested access group node to the + /// symbol pointing to the translated operation. Returns success if all + /// conversions succeed and failure otherwise. + LogicalResult processAccessGroupMetadata(const llvm::MDNode *node); + /// Converts all LLVM alias scopes and domains starting from `node` to MLIR + /// alias scope and domain operations and stores a mapping from every nested + /// alias scope or alias domain node to the symbol pointing to the translated + /// operation. Returns success if all conversions succeed and failure + /// otherwise. + LogicalResult processAliasScopeMetadata(const llvm::MDNode *node); /// Builder pointing at where the next instruction should be generated. OpBuilder builder; @@ -257,6 +294,8 @@ class ModuleImport { Operation *constantInsertionOp = nullptr; /// Operation to insert the next global after. Operation *globalInsertionOp = nullptr; + /// Operation to insert metadata operations into. + MetadataOp globalMetadataOp = nullptr; /// The current context. MLIRContext *context; /// The MLIR module being created. @@ -276,20 +315,18 @@ class ModuleImport { /// operations for all operations that return no result. All operations that /// return a result have a valueMapping entry instead. DenseMap noResultOpMapping; + /// Mapping between LLVM alias scope and domain metadata nodes and symbol + /// references to the LLVM dialect operations corresponding to these nodes. + DenseMap aliasScopeMapping; + /// Mapping between LLVM TBAA metadata nodes and symbol references to the LLVM + /// dialect TBAA operations corresponding to these nodes. + DenseMap tbaaMapping; /// The stateful type translator (contains named structs). LLVM::TypeFromLLVMIRTranslator typeTranslator; /// Stateful debug information importer. std::unique_ptr debugImporter; - /// A terminated MetadataOp where TBAA metadata operations - /// can be inserted. - MetadataOp tbaaMetadataOp{}; - /// Mapping between LLVM TBAA metadata nodes and symbol references - /// to the LLVMIR dialect TBAA operations corresponding to these - /// nodes. - DenseMap tbaaMapping; - /// A counter to be used as a unique suffix for symbols - /// defined by TBAA operations. - unsigned tbaaNodeCounter = 0; + /// Loop annotation importer. + std::unique_ptr loopAnnotationImporter; }; } // namespace LLVM diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h index 33f8ab7d066d..faca8fc5e4fb 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -15,8 +15,8 @@ #define MLIR_TARGET_LLVMIR_MODULETRANSLATION_H #include "mlir/IR/Operation.h" -#include "mlir/IR/Value.h" #include "mlir/IR/SymbolTable.h" +#include "mlir/IR/Value.h" #include "mlir/Target/LLVMIR/Export.h" #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h" #include "mlir/Target/LLVMIR/TypeToLLVM.h" @@ -40,6 +40,7 @@ namespace LLVM { namespace detail { class DebugTranslation; +class LoopAnnotationTranslation; } // namespace detail class DINodeAttr; @@ -119,28 +120,9 @@ class ModuleTranslation { /// in these blocks. void forgetMapping(Region ®ion); - /// Returns the LLVM metadata corresponding to a reference to an mlir LLVM - /// dialect access group operation. - llvm::MDNode *getAccessGroup(Operation &opInst, - SymbolRefAttr accessGroupRef) const; - - /// Returns the LLVM metadata corresponding to a reference to an mlir LLVM - /// dialect alias scope operation - llvm::MDNode *getAliasScope(Operation &opInst, - SymbolRefAttr aliasScopeRef) const; - - /// Returns the LLVM metadata corresponding to a llvm loop's codegen - /// options attribute. - llvm::MDNode *lookupLoopOptionsMetadata(Attribute options) const { - return loopOptionsMetadataMapping.lookup(options); - } - - void mapLoopOptionsMetadata(Attribute options, llvm::MDNode *metadata) { - auto result = loopOptionsMetadataMapping.try_emplace(options, metadata); - (void)result; - assert(result.second && - "attempting to map loop options that was already mapped"); - } + /// Returns the LLVM metadata corresponding to a symbol reference to an mlir + /// LLVM dialect alias scope operation + llvm::MDNode *getAliasScope(Operation *op, SymbolRefAttr aliasScopeRef) const; // Sets LLVM metadata for memory operations that are in a parallel loop. void setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst); @@ -152,6 +134,10 @@ class ModuleTranslation { /// TBAA attributes. void setTBAAMetadata(Operation *op, llvm::Instruction *inst); + /// Sets LLVM loop metadata for branch operations that have a loop annotation + /// attribute. + void setLoopMetadata(Operation *op, llvm::Instruction *inst); + /// Converts the type from MLIR LLVM dialect to LLVM. llvm::Type *convertType(Type type); @@ -273,7 +259,7 @@ class ModuleTranslation { ModuleTranslation &moduleTranslation; }; - SymbolTableCollection& symbolTable() { return symbolTableCollection; } + SymbolTableCollection &symbolTable() { return symbolTableCollection; } private: ModuleTranslation(Operation *module, @@ -295,9 +281,9 @@ class ModuleTranslation { /// metadata nodes for them and their domains. LogicalResult createAliasScopeMetadata(); - /// Returns the LLVM metadata corresponding to a reference to an mlir LLVM - /// dialect TBAATagOp operation. - llvm::MDNode *getTBAANode(Operation &memOp, SymbolRefAttr tagRef) const; + /// Returns the LLVM metadata corresponding to a symbol reference to an mlir + /// LLVM dialect TBAATagOp operation. + llvm::MDNode *getTBAANode(Operation *op, SymbolRefAttr tagRef) const; /// Process tbaa LLVM Metadata operations and create LLVM /// metadata nodes for them. @@ -306,12 +292,18 @@ class ModuleTranslation { /// Translates dialect attributes attached to the given operation. LogicalResult convertDialectAttributes(Operation *op); + /// Translates parameter attributes and adds them to the returned AttrBuilder. + llvm::AttrBuilder convertParameterAttrs(DictionaryAttr paramAttrs); + /// Original and translated module. Operation *mlirModule; std::unique_ptr llvmModule; /// A converter for translating debug information. std::unique_ptr debugTranslation; + /// A converter for translating loop annotations. + std::unique_ptr loopAnnotationTranslation; + /// Builder for LLVM IR generation of OpenMP constructs. std::unique_ptr ompBuilder; @@ -335,16 +327,6 @@ class ModuleTranslation { /// values after all operations are converted. DenseMap branchMapping; - /// Mapping from an access group metadata operation to its LLVM metadata. - /// This map is populated on module entry and is used to annotate loops (as - /// identified via their branches) and contained memory accesses. - DenseMap accessGroupMetadataMapping; - - /// Mapping from an attribute describing loop codegen options to its LLVM - /// metadata. The metadata is attached to Latch block branches with this - /// attribute. - DenseMap loopOptionsMetadataMapping; - /// Mapping from an alias scope metadata operation to its LLVM metadata. /// This map is populated on module entry. DenseMap aliasScopeMetadataMapping; diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td index e2a15934e3cf..f52cbfc482c5 100644 --- a/mlir/include/mlir/Transforms/Passes.td +++ b/mlir/include/mlir/Transforms/Passes.td @@ -277,7 +277,11 @@ def ViewOpGraph : Pass<"view-op-graph"> { Option<"printDataFlowEdges", "print-data-flow-edges", "bool", /*default=*/"true", "Print data flow edges">, Option<"printResultTypes", "print-result-types", "bool", - /*default=*/"true", "Print result types of operations"> + /*default=*/"true", "Print result types of operations">, + Option<"printRegionControlFlowEdges", "print-region-control-flow-edges", + "bool", /*default=*/"false", "Print region control flow edges">, + Option<"onlyEntryAndExitOperations", "only-entry-exit-ops", + "bool", /*default=*/"false", "Print only entry and exit operations"> ]; let constructor = "mlir::createPrintOpGraphPass()"; } diff --git a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp index 53e55470fcd3..1223a6c9da5a 100644 --- a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp +++ b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp @@ -268,6 +268,8 @@ class EncodingReader { return parseBytes(static_cast(length), sectionData); } + Location getLoc() const { return fileLoc; } + private: /// Parse a variable length encoded integer from the byte stream. This method /// is a fallback when the number of bytes used to encode the value is greater @@ -835,6 +837,10 @@ class DialectReader : public DialectBytecodeReader { return reader.emitError(msg); } + MLIRContext *getContext() const override { return getLoc().getContext(); } + + Location getLoc() const { return reader.getLoc(); } + //===--------------------------------------------------------------------===// // IR //===--------------------------------------------------------------------===// diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp index 5f00c3da77ea..628127757cdf 100644 --- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp +++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp @@ -24,7 +24,7 @@ #include "llvm/ADT/TypeSwitch.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTASYNCTOLLVM +#define GEN_PASS_DEF_CONVERTASYNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -378,7 +378,8 @@ class CoroBeginOpConversion : public OpConversionPattern { // Allocate memory for the coroutine frame. auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn( - op->getParentOfType(), rewriter.getI64Type()); + op->getParentOfType(), rewriter.getI64Type(), + /*TODO: opaquePointers=*/false); auto coroAlloc = rewriter.create( loc, allocFuncOp, ValueRange{coroAlign, coroSize}); @@ -412,8 +413,8 @@ class CoroFreeOpConversion : public OpConversionPattern { rewriter.create(loc, i8Ptr, adaptor.getOperands()); // Free the memory. - auto freeFuncOp = - LLVM::lookupOrCreateFreeFn(op->getParentOfType()); + auto freeFuncOp = LLVM::lookupOrCreateFreeFn( + op->getParentOfType(), /*TODO: opaquePointers=*/false); rewriter.replaceOpWithNewOp(op, freeFuncOp, ValueRange(coroMem.getResult())); @@ -990,7 +991,9 @@ class ReturnOpOpConversion : public OpConversionPattern { namespace { struct ConvertAsyncToLLVMPass - : public impl::ConvertAsyncToLLVMBase { + : public impl::ConvertAsyncToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -1121,10 +1124,6 @@ class ConvertYieldOpTypes : public OpConversionPattern { }; } // namespace -std::unique_ptr> mlir::createConvertAsyncToLLVMPass() { - return std::make_unique(); -} - void mlir::populateAsyncStructuralTypeConversionsAndLegality( TypeConverter &typeConverter, RewritePatternSet &patterns, ConversionTarget &target) { diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp index 14f32d66d244..7185b01afdc0 100644 --- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp +++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp @@ -16,7 +16,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTCOMPLEXTOLLVM +#define GEN_PASS_DEF_CONVERTCOMPLEXTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -323,7 +323,9 @@ void mlir::populateComplexToLLVMConversionPatterns( namespace { struct ConvertComplexToLLVMPass - : public impl::ConvertComplexToLLVMBase { + : public impl::ConvertComplexToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -340,7 +342,3 @@ void ConvertComplexToLLVMPass::runOnOperation() { applyPartialConversion(getOperation(), target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr mlir::createConvertComplexToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp index c51daeb05ac5..bcfbfdc20af3 100644 --- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp +++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp @@ -27,7 +27,7 @@ #include namespace mlir { -#define GEN_PASS_DEF_CONVERTCONTROLFLOWTOLLVM +#define GEN_PASS_DEF_CONVERTCONTROLFLOWTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -72,7 +72,8 @@ static void createPrintMsg(OpBuilder &builder, Location loc, ModuleOp moduleOp, SmallVector indices(1, 0); Value gep = builder.create( loc, LLVM::LLVMPointerType::get(builder.getI8Type()), msgAddr, indices); - Operation *printer = LLVM::lookupOrCreatePrintStrFn(moduleOp); + Operation *printer = + LLVM::lookupOrCreatePrintStrFn(moduleOp, /*TODO: opaquePointers=*/false); builder.create(loc, TypeRange(), SymbolRefAttr::get(printer), gep); } @@ -262,8 +263,9 @@ void mlir::cf::populateAssertToLLVMConversionPattern( namespace { /// A pass converting MLIR operations into the LLVM IR dialect. struct ConvertControlFlowToLLVM - : public impl::ConvertControlFlowToLLVMBase { - ConvertControlFlowToLLVM() = default; + : public impl::ConvertControlFlowToLLVMPassBase { + + using Base::Base; /// Run the dialect converter on the module. void runOnOperation() override { @@ -283,7 +285,3 @@ struct ConvertControlFlowToLLVM } }; } // namespace - -std::unique_ptr mlir::cf::createConvertControlFlowToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp index 6ff52566dee3..178b5f03e9a2 100644 --- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp +++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp @@ -48,7 +48,7 @@ #include namespace mlir { -#define GEN_PASS_DEF_CONVERTFUNCTOLLVM +#define GEN_PASS_DEF_CONVERTFUNCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -78,54 +78,26 @@ static void filterFuncAttributes(func::FuncOp func, bool filterArgAndResAttrs, } } -/// Helper function for wrapping all attributes into a single DictionaryAttr -static auto wrapAsStructAttrs(OpBuilder &b, ArrayAttr attrs) { - return DictionaryAttr::get( - b.getContext(), - b.getNamedAttr(LLVM::LLVMDialect::getStructAttrsAttrName(), attrs)); -} +/// Adds a an empty set of argument attributes for the newly added argument in +/// front of the existing ones. +static void prependEmptyArgAttr(OpBuilder &builder, + SmallVectorImpl &newFuncAttrs, + func::FuncOp func) { + auto argAttrs = func.getArgAttrs(); + // Nothing to do when there were no arg attrs beforehand. + if (!argAttrs) + return; -/// Combines all result attributes into a single DictionaryAttr -/// and prepends to argument attrs. -/// This is intended to be used to format the attributes for a C wrapper -/// function when the result(s) is converted to the first function argument -/// (in the multiple return case, all returns get wrapped into a single -/// argument). The total number of argument attributes should be equal to -/// (number of function arguments) + 1. -static void -prependResAttrsToArgAttrs(OpBuilder &builder, - SmallVectorImpl &attributes, - func::FuncOp func) { size_t numArguments = func.getNumArguments(); - auto allAttrs = SmallVector( - numArguments + 1, DictionaryAttr::get(builder.getContext())); - NamedAttribute *argAttrs = nullptr; - for (auto *it = attributes.begin(); it != attributes.end();) { - if (it->getName() == func.getArgAttrsAttrName()) { - auto arrayAttrs = it->getValue().cast(); - assert(arrayAttrs.size() == numArguments && - "Number of arg attrs and args should match"); - std::copy(arrayAttrs.begin(), arrayAttrs.end(), allAttrs.begin() + 1); - argAttrs = it; - } else if (it->getName() == func.getResAttrsAttrName()) { - auto arrayAttrs = it->getValue().cast(); - assert(!arrayAttrs.empty() && "expected array to be non-empty"); - allAttrs[0] = (arrayAttrs.size() == 1) - ? arrayAttrs[0] - : wrapAsStructAttrs(builder, arrayAttrs); - it = attributes.erase(it); - continue; - } - it++; - } - - auto newArgAttrs = builder.getNamedAttr(func.getArgAttrsAttrName(), - builder.getArrayAttr(allAttrs)); - if (!argAttrs) { - attributes.emplace_back(newArgAttrs); - return; - } - *argAttrs = newArgAttrs; + SmallVector newArgAttrs; + newArgAttrs.reserve(numArguments + 1); + // Insert empty dictionary for the new argument. + newArgAttrs.push_back(builder.getDictionaryAttr({})); + + llvm::append_range(newArgAttrs, *argAttrs); + auto newNamedAttr = builder.getNamedAttr(func.getArgAttrsAttrName(), + builder.getArrayAttr(newArgAttrs)); + newFuncAttrs.push_back(newNamedAttr); } /// Creates an auxiliary function with pointer-to-memref-descriptor-struct @@ -141,12 +113,16 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc, func::FuncOp funcOp, LLVM::LLVMFuncOp newFuncOp) { auto type = funcOp.getFunctionType(); - SmallVector attributes; - filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/false, attributes); auto [wrapperFuncType, resultIsNowArg] = typeConverter.convertFunctionTypeCWrapper(type); + + SmallVector attributes; + // Only modify the argument and result attributes when the result is now an + // argument. if (resultIsNowArg) - prependResAttrsToArgAttrs(rewriter, attributes, funcOp); + prependEmptyArgAttr(rewriter, attributes, funcOp); + filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/resultIsNowArg, + attributes); auto wrapperFuncOp = rewriter.create( loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(), wrapperFuncType, LLVM::Linkage::External, /*dsoLocal*/ false, @@ -207,10 +183,13 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc, assert(wrapperType && "unexpected type conversion failure"); SmallVector attributes; - filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/false, attributes); - + // Only modify the argument and result attributes when the result is now an + // argument. if (resultIsNowArg) - prependResAttrsToArgAttrs(builder, attributes, funcOp); + prependEmptyArgAttr(builder, attributes, funcOp); + filterFuncAttributes(funcOp, /*filterArgAndResAttrs=*/resultIsNowArg, + attributes); + // Create the auxiliary function. auto wrapperFunc = builder.create( loc, llvm::formatv("_mlir_ciface_{0}", funcOp.getName()).str(), @@ -312,8 +291,7 @@ struct FuncOpConversionBase : public ConvertOpToLLVMPattern { auto newResAttrDicts = (funcOp.getNumResults() == 1) ? resAttrDicts - : rewriter.getArrayAttr( - {wrapAsStructAttrs(rewriter, resAttrDicts)}); + : rewriter.getArrayAttr(rewriter.getDictionaryAttr({})); attributes.push_back( rewriter.getNamedAttr(funcOp.getResAttrsAttrName(), newResAttrDicts)); } @@ -727,15 +705,8 @@ void mlir::populateFuncToLLVMConversionPatterns(LLVMTypeConverter &converter, namespace { /// A pass converting Func operations into the LLVM IR dialect. struct ConvertFuncToLLVMPass - : public impl::ConvertFuncToLLVMBase { - ConvertFuncToLLVMPass() = default; - ConvertFuncToLLVMPass(bool useBarePtrCallConv, unsigned indexBitwidth, - bool useAlignedAlloc, - const llvm::DataLayout &dataLayout) { - this->useBarePtrCallConv = useBarePtrCallConv; - this->indexBitwidth = indexBitwidth; - this->dataLayout = dataLayout.getStringRepresentation(); - } + : public impl::ConvertFuncToLLVMPassBase { + using Base::Base; /// Run the dialect converter on the module. void runOnOperation() override { @@ -776,21 +747,3 @@ struct ConvertFuncToLLVMPass } }; } // namespace - -std::unique_ptr> mlir::createConvertFuncToLLVMPass() { - return std::make_unique(); -} - -std::unique_ptr> -mlir::createConvertFuncToLLVMPass(const LowerToLLVMOptions &options) { - auto allocLowering = options.allocLowering; - // There is no way to provide additional patterns for pass, so - // AllocLowering::None will always fail. - assert(allocLowering != LowerToLLVMOptions::AllocLowering::None && - "ConvertFuncToLLVMPass doesn't support AllocLowering::None"); - bool useAlignedAlloc = - (allocLowering == LowerToLLVMOptions::AllocLowering::AlignedAlloc); - return std::make_unique( - options.useBarePtrCallConv, options.getIndexBitwidth(), useAlignedAlloc, - options.dataLayout); -} diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp index c4dd47a762cf..4bb0e3ae028f 100644 --- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp +++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp @@ -52,30 +52,10 @@ namespace { class GpuToLLVMConversionPass : public impl::GpuToLLVMConversionPassBase { public: - GpuToLLVMConversionPass() = default; - - GpuToLLVMConversionPass(bool kernelBarePtrCallConv) - : GpuToLLVMConversionPass() { - if (this->kernelBarePtrCallConv.getNumOccurrences() == 0) - this->kernelBarePtrCallConv = kernelBarePtrCallConv; - } - - GpuToLLVMConversionPass(const GpuToLLVMConversionPass &other) - : GpuToLLVMConversionPassBase(other) {} + using Base::Base; // Run the dialect converter on the module. void runOnOperation() override; - -private: - Option gpuBinaryAnnotation{ - *this, "gpu-binary-annotation", - llvm::cl::desc("Annotation attribute string for GPU binary"), - llvm::cl::init(gpu::getDefaultGpuBinaryAnnotation())}; - Option kernelBarePtrCallConv{ - *this, "use-bare-pointers-for-kernels", - llvm::cl::desc("Use bare pointers to pass memref arguments to kernels. " - "The kernel must use the same setting for this option."), - llvm::cl::init(false)}; }; struct FunctionCallBuilder { @@ -112,7 +92,7 @@ class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern { MLIRContext *context = &this->getTypeConverter()->getContext(); Type llvmVoidType = LLVM::LLVMVoidType::get(context); - Type llvmPointerType = + LLVM::LLVMPointerType llvmPointerType = LLVM::LLVMPointerType::get(IntegerType::get(context, 8)); Type llvmPointerPointerType = LLVM::LLVMPointerType::get(llvmPointerType); Type llvmInt8Type = IntegerType::get(context, 8); @@ -392,7 +372,7 @@ void GpuToLLVMConversionPass::runOnOperation() { mlir::arith::populateArithToLLVMConversionPatterns(converter, patterns); mlir::cf::populateControlFlowToLLVMConversionPatterns(converter, patterns); populateVectorToLLVMConversionPatterns(converter, patterns); - populateMemRefToLLVMConversionPatterns(converter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, patterns); populateFuncToLLVMConversionPatterns(converter, patterns); populateAsyncStructuralTypeConversionsAndLegality(converter, patterns, target); @@ -827,6 +807,22 @@ LogicalResult ConvertLaunchFuncOpToGpuRuntimeCallPattern::matchAndRewrite( return success(); } +static Value bitAndAddrspaceCast(Location loc, + ConversionPatternRewriter &rewriter, + LLVM::LLVMPointerType destinationType, + Value sourcePtr, + LLVMTypeConverter &typeConverter) { + auto sourceTy = sourcePtr.getType().cast(); + if (destinationType.getAddressSpace() != sourceTy.getAddressSpace()) + sourcePtr = rewriter.create( + loc, + typeConverter.getPointerType(sourceTy.getElementType(), + destinationType.getAddressSpace()), + sourcePtr); + + return rewriter.create(loc, destinationType, sourcePtr); +} + LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite( gpu::MemcpyOp memcpyOp, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const { @@ -849,11 +845,13 @@ LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite( auto sizeBytes = rewriter.create(loc, getIndexType(), gepPtr); - auto src = rewriter.create( - loc, llvmPointerType, srcDesc.alignedPtr(rewriter, loc)); - auto dst = rewriter.create( - loc, llvmPointerType, - MemRefDescriptor(adaptor.getDst()).alignedPtr(rewriter, loc)); + auto src = bitAndAddrspaceCast(loc, rewriter, llvmPointerType, + srcDesc.alignedPtr(rewriter, loc), + *getTypeConverter()); + auto dst = bitAndAddrspaceCast( + loc, rewriter, llvmPointerType, + MemRefDescriptor(adaptor.getDst()).alignedPtr(rewriter, loc), + *getTypeConverter()); auto stream = adaptor.getAsyncDependencies().front(); memcpyCallBuilder.create(loc, rewriter, {dst, src, sizeBytes, stream}); @@ -886,8 +884,9 @@ LogicalResult ConvertMemsetOpToGpuRuntimeCallPattern::matchAndRewrite( auto value = rewriter.create(loc, llvmInt32Type, adaptor.getValue()); - auto dst = rewriter.create( - loc, llvmPointerType, dstDesc.alignedPtr(rewriter, loc)); + auto dst = bitAndAddrspaceCast(loc, rewriter, llvmPointerType, + dstDesc.alignedPtr(rewriter, loc), + *getTypeConverter()); auto stream = adaptor.getAsyncDependencies().front(); memsetCallBuilder.create(loc, rewriter, {dst, value, numElements, stream}); @@ -905,11 +904,6 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite( return success(); } -std::unique_ptr> -mlir::createGpuToLLVMConversionPass(bool kernelBarePtrCallConv) { - return std::make_unique(kernelBarePtrCallConv); -} - void mlir::populateGpuToLLVMConversionPatterns(LLVMTypeConverter &converter, RewritePatternSet &patterns, StringRef gpuBinaryAnnotation, diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp index cefe493ecbc0..a80c024ceb24 100644 --- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp +++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp @@ -281,7 +281,7 @@ struct LowerGpuOpsToNVVMOpsPass arith::populateArithToLLVMConversionPatterns(converter, llvmPatterns); cf::populateControlFlowToLLVMConversionPatterns(converter, llvmPatterns); populateFuncToLLVMConversionPatterns(converter, llvmPatterns); - populateMemRefToLLVMConversionPatterns(converter, llvmPatterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, llvmPatterns); populateGpuToNVVMConversionPatterns(converter, llvmPatterns); populateGpuWMMAToNVVMConversionPatterns(converter, llvmPatterns); if (this->hasRedux) diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp index daf6583ef6e6..233e2fb3681d 100644 --- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp +++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp @@ -167,7 +167,7 @@ struct LowerGpuOpsToROCDLOpsPass populateVectorToLLVMConversionPatterns(converter, llvmPatterns); cf::populateControlFlowToLLVMConversionPatterns(converter, llvmPatterns); populateFuncToLLVMConversionPatterns(converter, llvmPatterns); - populateMemRefToLLVMConversionPatterns(converter, llvmPatterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, llvmPatterns); populateGpuToROCDLConversionPatterns(converter, llvmPatterns, runtime); LLVMConversionTarget target(getContext()); configureGpuToROCDLConversionLegality(target); diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp index f808e3dabf5c..8ce9fb69ee6d 100644 --- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp +++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp @@ -132,22 +132,30 @@ Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) { Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos, int64_t rank) { - auto indexPtrTy = LLVM::LLVMPointerType::get(indexType); auto arrayTy = LLVM::LLVMArrayType::get(indexType, rank); - auto arrayPtrTy = LLVM::LLVMPointerType::get(arrayTy); + + LLVM::LLVMPointerType indexPtrTy; + LLVM::LLVMPointerType arrayPtrTy; + + if (useOpaquePointers()) { + arrayPtrTy = indexPtrTy = LLVM::LLVMPointerType::get(builder.getContext()); + } else { + indexPtrTy = LLVM::LLVMPointerType::get(indexType); + arrayPtrTy = LLVM::LLVMPointerType::get(arrayTy); + } // Copy size values to stack-allocated memory. auto one = createIndexAttrConstant(builder, loc, indexType, 1); auto sizes = builder.create( loc, value, llvm::ArrayRef({kSizePosInMemRefDescriptor})); - auto sizesPtr = - builder.create(loc, arrayPtrTy, one, /*alignment=*/0); + auto sizesPtr = builder.create(loc, arrayPtrTy, arrayTy, one, + /*alignment=*/0); builder.create(loc, sizes, sizesPtr); // Load an return size value of interest. - auto resultPtr = builder.create(loc, indexPtrTy, sizesPtr, - ArrayRef{0, pos}); - return builder.create(loc, resultPtr); + auto resultPtr = builder.create( + loc, indexPtrTy, arrayTy, sizesPtr, ArrayRef{0, pos}); + return builder.create(loc, indexType, resultPtr); } /// Builds IR inserting the pos-th size into the descriptor @@ -242,6 +250,10 @@ unsigned MemRefDescriptor::getNumUnpackedValues(MemRefType type) { return 3 + 2 * type.getRank(); } +bool MemRefDescriptor::useOpaquePointers() { + return getElementPtrType().isOpaque(); +} + //===----------------------------------------------------------------------===// // MemRefDescriptorView implementation. //===----------------------------------------------------------------------===// @@ -372,134 +384,195 @@ void UnrankedMemRefDescriptor::computeSizes( } } -Value UnrankedMemRefDescriptor::allocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, - Type elemPtrPtrType) { +Value UnrankedMemRefDescriptor::allocatedPtr( + OpBuilder &builder, Location loc, Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType) { + + Value elementPtrPtr; + if (elemPtrType.isOpaque()) + elementPtrPtr = memRefDescPtr; + else + elementPtrPtr = builder.create( + loc, LLVM::LLVMPointerType::get(elemPtrType), memRefDescPtr); - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); - return builder.create(loc, elementPtrPtr); + return builder.create(loc, elemPtrType, elementPtrPtr); } -void UnrankedMemRefDescriptor::setAllocatedPtr(OpBuilder &builder, Location loc, - Value memRefDescPtr, - Type elemPtrPtrType, - Value allocatedPtr) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); +void UnrankedMemRefDescriptor::setAllocatedPtr( + OpBuilder &builder, Location loc, Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType, Value allocatedPtr) { + Value elementPtrPtr; + if (elemPtrType.isOpaque()) + elementPtrPtr = memRefDescPtr; + else + elementPtrPtr = builder.create( + loc, LLVM::LLVMPointerType::get(elemPtrType), memRefDescPtr); + builder.create(loc, allocatedPtr, elementPtrPtr); } +static std::pair +castToElemPtrPtr(OpBuilder &builder, Location loc, Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType) { + Value elementPtrPtr; + Type elemPtrPtrType; + if (elemPtrType.isOpaque()) { + elementPtrPtr = memRefDescPtr; + elemPtrPtrType = LLVM::LLVMPointerType::get(builder.getContext()); + } else { + elemPtrPtrType = LLVM::LLVMPointerType::get(elemPtrType); + elementPtrPtr = + builder.create(loc, elemPtrPtrType, memRefDescPtr); + } + return {elementPtrPtr, elemPtrPtrType}; +} + Value UnrankedMemRefDescriptor::alignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); + LLVM::LLVMPointerType elemPtrType) { + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); - Value alignedGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{1}); - return builder.create(loc, alignedGep); + Value alignedGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{1}); + return builder.create(loc, elemPtrType, alignedGep); } void UnrankedMemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType, + LLVM::LLVMPointerType elemPtrType, Value alignedPtr) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); - Value alignedGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{1}); + Value alignedGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{1}); builder.create(loc, alignedPtr, alignedGep); } Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); + LLVM::LLVMPointerType elemPtrType) { + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); + + Value offsetGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{2}); + + if (!elemPtrType.isOpaque()) { + offsetGep = builder.create( + loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), + offsetGep); + } - Value offsetGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{2}); - offsetGep = builder.create( - loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), offsetGep); - return builder.create(loc, offsetGep); + return builder.create(loc, typeConverter.getIndexType(), + offsetGep); } void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value memRefDescPtr, - Type elemPtrPtrType, Value offset) { - Value elementPtrPtr = - builder.create(loc, elemPtrPtrType, memRefDescPtr); - - Value offsetGep = builder.create( - loc, elemPtrPtrType, elementPtrPtr, ArrayRef{2}); - offsetGep = builder.create( - loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), offsetGep); + LLVM::LLVMPointerType elemPtrType, + Value offset) { + auto [elementPtrPtr, elemPtrPtrType] = + castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType); + + Value offsetGep = + builder.create(loc, elemPtrPtrType, elemPtrType, + elementPtrPtr, ArrayRef{2}); + + if (!elemPtrType.isOpaque()) { + offsetGep = builder.create( + loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()), + offsetGep); + } + builder.create(loc, offset, offsetGep); } -Value UnrankedMemRefDescriptor::sizeBasePtr( - OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, - Value memRefDescPtr, LLVM::LLVMPointerType elemPtrPtrType) { - Type elemPtrTy = elemPtrPtrType.getElementType(); +Value UnrankedMemRefDescriptor::sizeBasePtr(OpBuilder &builder, Location loc, + LLVMTypeConverter &typeConverter, + Value memRefDescPtr, + LLVM::LLVMPointerType elemPtrType) { Type indexTy = typeConverter.getIndexType(); - Type structPtrTy = - LLVM::LLVMPointerType::get(LLVM::LLVMStructType::getLiteral( - indexTy.getContext(), {elemPtrTy, elemPtrTy, indexTy, indexTy})); - Value structPtr = - builder.create(loc, structPtrTy, memRefDescPtr); + Type structTy = LLVM::LLVMStructType::getLiteral( + indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy}); + Value structPtr; + if (elemPtrType.isOpaque()) { + structPtr = memRefDescPtr; + } else { + Type structPtrTy = LLVM::LLVMPointerType::get(structTy); + structPtr = + builder.create(loc, structPtrTy, memRefDescPtr); + } - return builder.create(loc, LLVM::LLVMPointerType::get(indexTy), - structPtr, ArrayRef{0, 3}); + auto resultType = elemPtrType.isOpaque() + ? LLVM::LLVMPointerType::get(indexTy.getContext()) + : LLVM::LLVMPointerType::get(indexTy); + return builder.create(loc, resultType, structTy, structPtr, + ArrayRef{0, 3}); } Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, Value index) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); + + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + Value sizeStoreGep = - builder.create(loc, indexPtrTy, sizeBasePtr, index); - return builder.create(loc, sizeStoreGep); + builder.create(loc, indexPtrTy, indexTy, sizeBasePtr, index); + return builder.create(loc, indexTy, sizeStoreGep); } void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, Value index, Value size) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + Value sizeStoreGep = - builder.create(loc, indexPtrTy, sizeBasePtr, index); + builder.create(loc, indexPtrTy, indexTy, sizeBasePtr, index); builder.create(loc, size, sizeStoreGep); } Value UnrankedMemRefDescriptor::strideBasePtr(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value sizeBasePtr, Value rank) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); - return builder.create(loc, indexPtrTy, sizeBasePtr, rank); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + + return builder.create(loc, indexPtrTy, indexTy, sizeBasePtr, + rank); } Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value strideBasePtr, Value index, Value stride) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); - Value strideStoreGep = - builder.create(loc, indexPtrTy, strideBasePtr, index); - return builder.create(loc, strideStoreGep); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + + Value strideStoreGep = builder.create(loc, indexPtrTy, indexTy, + strideBasePtr, index); + return builder.create(loc, indexTy, strideStoreGep); } void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter, Value strideBasePtr, Value index, Value stride) { - Type indexPtrTy = LLVM::LLVMPointerType::get(typeConverter.getIndexType()); - Value strideStoreGep = - builder.create(loc, indexPtrTy, strideBasePtr, index); + Type indexTy = typeConverter.getIndexType(); + Type indexPtrTy = typeConverter.getPointerType(indexTy); + + Value strideStoreGep = builder.create(loc, indexPtrTy, indexTy, + strideBasePtr, index); builder.create(loc, stride, strideStoreGep); } diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp index 6c28022e1a07..f66f2e21c705 100644 --- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp +++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp @@ -48,7 +48,7 @@ Type ConvertToLLVMPattern::getVoidType() const { } Type ConvertToLLVMPattern::getVoidPtrType() const { - return LLVM::LLVMPointerType::get( + return getTypeConverter()->getPointerType( IntegerType::get(&getTypeConverter()->getContext(), 8)); } @@ -93,7 +93,10 @@ Value ConvertToLLVMPattern::getStridedElementPtr( } Type elementPtrType = memRefDescriptor.getElementPtrType(); - return index ? rewriter.create(loc, elementPtrType, base, index) + return index ? rewriter.create( + loc, elementPtrType, + getTypeConverter()->convertType(type.getElementType()), + base, index) : base; } @@ -109,8 +112,8 @@ bool ConvertToLLVMPattern::isConvertibleAndHasIdentityMaps( Type ConvertToLLVMPattern::getElementPtrType(MemRefType type) const { auto elementType = type.getElementType(); auto structElementType = typeConverter->convertType(elementType); - return LLVM::LLVMPointerType::get(structElementType, - type.getMemorySpaceAsInt()); + return getTypeConverter()->getPointerType(structElementType, + type.getMemorySpaceAsInt()); } void ConvertToLLVMPattern::getMemRefDescriptorSizes( @@ -157,10 +160,11 @@ void ConvertToLLVMPattern::getMemRefDescriptorSizes( } // Buffer size in bytes. - Type elementPtrType = getElementPtrType(memRefType); + Type elementType = typeConverter->convertType(memRefType.getElementType()); + Type elementPtrType = getTypeConverter()->getPointerType(elementType); Value nullPtr = rewriter.create(loc, elementPtrType); - Value gepPtr = - rewriter.create(loc, elementPtrType, nullPtr, runningStride); + Value gepPtr = rewriter.create(loc, elementPtrType, elementType, + nullPtr, runningStride); sizeBytes = rewriter.create(loc, getIndexType(), gepPtr); } @@ -171,11 +175,11 @@ Value ConvertToLLVMPattern::getSizeInBytes( // %0 = getelementptr %elementType* null, %indexType 1 // %1 = ptrtoint %elementType* %0 to %indexType // which is a common pattern of getting the size of a type in bytes. - auto convertedPtrType = - LLVM::LLVMPointerType::get(typeConverter->convertType(type)); + Type llvmType = typeConverter->convertType(type); + auto convertedPtrType = getTypeConverter()->getPointerType(llvmType); auto nullPtr = rewriter.create(loc, convertedPtrType); - auto gep = rewriter.create(loc, convertedPtrType, nullPtr, - ArrayRef{1}); + auto gep = rewriter.create(loc, convertedPtrType, llvmType, + nullPtr, ArrayRef{1}); return rewriter.create(loc, getIndexType(), gep); } @@ -241,7 +245,6 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors( // Get frequently used types. MLIRContext *context = builder.getContext(); - Type voidPtrType = LLVM::LLVMPointerType::get(IntegerType::get(context, 8)); auto i1Type = IntegerType::get(context, 1); Type indexType = getTypeConverter()->getIndexType(); @@ -249,9 +252,11 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors( auto module = builder.getInsertionPoint()->getParentOfType(); LLVM::LLVMFuncOp freeFunc, mallocFunc; if (toDynamic) - mallocFunc = LLVM::lookupOrCreateMallocFn(module, indexType); + mallocFunc = LLVM::lookupOrCreateMallocFn( + module, indexType, getTypeConverter()->useOpaquePointers()); if (!toDynamic) - freeFunc = LLVM::lookupOrCreateFreeFn(module); + freeFunc = LLVM::lookupOrCreateFreeFn( + module, getTypeConverter()->useOpaquePointers()); // Initialize shared constants. Value zero = @@ -270,7 +275,8 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors( toDynamic ? builder.create(loc, mallocFunc, allocationSize) .getResult() - : builder.create(loc, voidPtrType, allocationSize, + : builder.create(loc, getVoidPtrType(), + getVoidType(), allocationSize, /*alignment=*/0); Value source = desc.memRefDescPtr(builder, loc); builder.create(loc, memory, source, allocationSize, zero); diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp index f5b35b794bf1..84496d998457 100644 --- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp +++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp @@ -169,6 +169,13 @@ Type LLVMTypeConverter::getIndexType() { return IntegerType::get(&getContext(), getIndexTypeBitwidth()); } +LLVM::LLVMPointerType +LLVMTypeConverter::getPointerType(Type elementType, unsigned int addressSpace) { + if (useOpaquePointers()) + return LLVM::LLVMPointerType::get(&getContext(), addressSpace); + return LLVM::LLVMPointerType::get(elementType, addressSpace); +} + unsigned LLVMTypeConverter::getPointerBitwidth(unsigned addressSpace) { return options.dataLayout.getPointerSizeInBits(addressSpace); } @@ -201,7 +208,7 @@ Type LLVMTypeConverter::convertFunctionType(FunctionType type) { convertFunctionSignature(type, /*isVariadic=*/false, conversion); if (!converted) return {}; - return LLVM::LLVMPointerType::get(converted); + return getPointerType(converted); } // Function types are converted to LLVM Function types by recursively converting @@ -311,8 +318,9 @@ LLVMTypeConverter::getMemRefDescriptorFields(MemRefType type, Type elementType = convertType(type.getElementType()); if (!elementType) return {}; - auto ptrTy = - LLVM::LLVMPointerType::get(elementType, type.getMemorySpaceAsInt()); + + LLVM::LLVMPointerType ptrTy = + getPointerType(elementType, type.getMemorySpaceAsInt()); auto indexTy = getIndexType(); SmallVector results = {ptrTy, ptrTy, indexTy}; @@ -355,8 +363,7 @@ Type LLVMTypeConverter::convertMemRefType(MemRefType type) { /// stack allocated (alloca) copy of a MemRef descriptor that got casted to /// be unranked. SmallVector LLVMTypeConverter::getUnrankedMemRefDescriptorFields() { - return {getIndexType(), - LLVM::LLVMPointerType::get(IntegerType::get(&getContext(), 8))}; + return {getIndexType(), getPointerType(IntegerType::get(&getContext(), 8))}; } unsigned @@ -406,7 +413,7 @@ Type LLVMTypeConverter::convertMemRefToBarePtr(BaseMemRefType type) { Type elementType = convertType(type.getElementType()); if (!elementType) return {}; - return LLVM::LLVMPointerType::get(elementType, type.getMemorySpaceAsInt()); + return getPointerType(elementType, type.getMemorySpaceAsInt()); } /// Convert an n-D vector type to an LLVM vector type: @@ -483,11 +490,11 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand, OpBuilder &builder) { // Alloca with proper alignment. We do not expect optimizations of this // alloca op and so we omit allocating at the entry block. - auto ptrType = LLVM::LLVMPointerType::get(operand.getType()); + auto ptrType = getPointerType(operand.getType()); Value one = builder.create(loc, builder.getI64Type(), builder.getIndexAttr(1)); Value allocated = - builder.create(loc, ptrType, one, /*alignment=*/0); + builder.create(loc, ptrType, operand.getType(), one); // Store into the alloca'ed descriptor. builder.create(loc, operand, allocated); return allocated; diff --git a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp index 7bc3680c2eae..8eb7bbe3e503 100644 --- a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp +++ b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp @@ -41,7 +41,7 @@ #include "llvm/Support/ErrorHandling.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTLINALGTOLLVM +#define GEN_PASS_DEF_CONVERTLINALGTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -78,7 +78,7 @@ void mlir::populateLinalgToLLVMConversionPatterns(LLVMTypeConverter &converter, namespace { struct ConvertLinalgToLLVMPass - : public impl::ConvertLinalgToLLVMBase { + : public impl::ConvertLinalgToLLVMPassBase { void runOnOperation() override; }; } // namespace @@ -90,14 +90,10 @@ void ConvertLinalgToLLVMPass::runOnOperation() { RewritePatternSet patterns(&getContext()); LLVMTypeConverter converter(&getContext()); populateLinalgToLLVMConversionPatterns(converter, patterns); - populateMemRefToLLVMConversionPatterns(converter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, patterns); LLVMConversionTarget target(getContext()); target.addLegalOp(); if (failed(applyPartialConversion(module, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> mlir::createConvertLinalgToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp index ece80921e29b..888c51238d06 100644 --- a/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp +++ b/mlir/lib/Conversion/MathToLLVM/MathToLLVM.cpp @@ -18,7 +18,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTMATHTOLLVM +#define GEN_PASS_DEF_CONVERTMATHTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -285,8 +285,8 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern { }; struct ConvertMathToLLVMPass - : public impl::ConvertMathToLLVMBase { - ConvertMathToLLVMPass() = default; + : public impl::ConvertMathToLLVMPassBase { + using Base::Base; void runOnOperation() override { RewritePatternSet patterns(&getContext()); @@ -332,7 +332,3 @@ void mlir::populateMathToLLVMConversionPatterns(LLVMTypeConverter &converter, >(converter); // clang-format on } - -std::unique_ptr mlir::createConvertMathToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp index 4a5be4870709..4afe2ffebb34 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/AllocLikeConversion.cpp @@ -23,9 +23,11 @@ LLVM::LLVMFuncOp getNotalignedAllocFn(LLVMTypeConverter *typeConverter, bool useGenericFn = typeConverter->getOptions().useGenericFunctions; if (useGenericFn) - return LLVM::lookupOrCreateGenericAllocFn(module, indexType); + return LLVM::lookupOrCreateGenericAllocFn( + module, indexType, typeConverter->useOpaquePointers()); - return LLVM::lookupOrCreateMallocFn(module, indexType); + return LLVM::lookupOrCreateMallocFn(module, indexType, + typeConverter->useOpaquePointers()); } LLVM::LLVMFuncOp getAlignedAllocFn(LLVMTypeConverter *typeConverter, @@ -33,9 +35,11 @@ LLVM::LLVMFuncOp getAlignedAllocFn(LLVMTypeConverter *typeConverter, bool useGenericFn = typeConverter->getOptions().useGenericFunctions; if (useGenericFn) - return LLVM::lookupOrCreateGenericAlignedAllocFn(module, indexType); + return LLVM::lookupOrCreateGenericAlignedAllocFn( + module, indexType, typeConverter->useOpaquePointers()); - return LLVM::lookupOrCreateAlignedAllocFn(module, indexType); + return LLVM::lookupOrCreateAlignedAllocFn(module, indexType, + typeConverter->useOpaquePointers()); } } // end namespace @@ -50,6 +54,24 @@ Value AllocationOpLLVMLowering::createAligned( return rewriter.create(loc, bumped, mod); } +static Value castAllocFuncResult(ConversionPatternRewriter &rewriter, + Location loc, Value allocatedPtr, + MemRefType memRefType, Type elementPtrType, + LLVMTypeConverter &typeConverter) { + auto allocatedPtrTy = allocatedPtr.getType().cast(); + if (allocatedPtrTy.getAddressSpace() != memRefType.getMemorySpaceAsInt()) + allocatedPtr = rewriter.create( + loc, + typeConverter.getPointerType(allocatedPtrTy.getElementType(), + memRefType.getMemorySpaceAsInt()), + allocatedPtr); + + if (!typeConverter.useOpaquePointers()) + allocatedPtr = + rewriter.create(loc, elementPtrType, allocatedPtr); + return allocatedPtr; +} + std::tuple AllocationOpLLVMLowering::allocateBufferManuallyAlign( ConversionPatternRewriter &rewriter, Location loc, Value sizeBytes, Operation *op, Value alignment) const { @@ -64,8 +86,10 @@ std::tuple AllocationOpLLVMLowering::allocateBufferManuallyAlign( LLVM::LLVMFuncOp allocFuncOp = getNotalignedAllocFn( getTypeConverter(), op->getParentOfType(), getIndexType()); auto results = rewriter.create(loc, allocFuncOp, sizeBytes); - Value allocatedPtr = rewriter.create(loc, elementPtrType, - results.getResult()); + + Value allocatedPtr = + castAllocFuncResult(rewriter, loc, results.getResult(), memRefType, + elementPtrType, *getTypeConverter()); Value alignedPtr = allocatedPtr; if (alignment) { @@ -126,10 +150,9 @@ Value AllocationOpLLVMLowering::allocateBufferAutoAlign( getTypeConverter(), op->getParentOfType(), getIndexType()); auto results = rewriter.create( loc, allocFuncOp, ValueRange({allocAlignment, sizeBytes})); - Value allocatedPtr = rewriter.create(loc, elementPtrType, - results.getResult()); - return allocatedPtr; + return castAllocFuncResult(rewriter, loc, results.getResult(), memRefType, + elementPtrType, *getTypeConverter()); } LogicalResult AllocLikeOpLLVMLowering::matchAndRewrite( diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp index 7132560ad8b1..d6097f057db4 100644 --- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp +++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp @@ -25,7 +25,7 @@ #include namespace mlir { -#define GEN_PASS_DEF_MEMREFTOLLVMCONVERSIONPASS +#define GEN_PASS_DEF_FINALIZEMEMREFTOLLVMCONVERSIONPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -41,9 +41,10 @@ LLVM::LLVMFuncOp getFreeFn(LLVMTypeConverter *typeConverter, ModuleOp module) { bool useGenericFn = typeConverter->getOptions().useGenericFunctions; if (useGenericFn) - return LLVM::lookupOrCreateGenericFreeFn(module); + return LLVM::lookupOrCreateGenericFreeFn( + module, typeConverter->useOpaquePointers()); - return LLVM::lookupOrCreateFreeFn(module); + return LLVM::lookupOrCreateFreeFn(module, typeConverter->useOpaquePointers()); } struct AllocOpLowering : public AllocLikeOpLLVMLowering { @@ -93,10 +94,14 @@ struct AllocaOpLowering : public AllocLikeOpLLVMLowering { // With alloca, one gets a pointer to the element type right away. // For stack allocations. auto allocaOp = cast(op); - auto elementPtrType = this->getElementPtrType(allocaOp.getType()); + auto elementType = + typeConverter->convertType(allocaOp.getType().getElementType()); + auto elementPtrType = getTypeConverter()->getPointerType( + elementType, allocaOp.getType().getMemorySpaceAsInt()); auto allocatedElementPtr = rewriter.create( - loc, elementPtrType, sizeBytes, allocaOp.getAlignment().value_or(0)); + loc, elementPtrType, elementType, sizeBytes, + allocaOp.getAlignment().value_or(0)); return std::make_tuple(allocatedElementPtr, allocatedElementPtr); } @@ -205,6 +210,8 @@ struct ReallocOpLoweringBase : public AllocationOpLLVMLowering { Value isVolatile = rewriter.create(loc, rewriter.getBoolAttr(false)); auto toVoidPtr = [&](Value ptr) -> Value { + if (getTypeConverter()->useOpaquePointers()) + return ptr; return rewriter.create(loc, getVoidPtrType(), ptr); }; rewriter.create(loc, toVoidPtr(dstAlignedPtr), @@ -372,9 +379,12 @@ struct DeallocOpLowering : public ConvertOpToLLVMPattern { LLVM::LLVMFuncOp freeFunc = getFreeFn(getTypeConverter(), op->getParentOfType()); MemRefDescriptor memref(adaptor.getMemref()); - Value casted = rewriter.create( - op.getLoc(), getVoidPtrType(), - memref.allocatedPtr(rewriter, op.getLoc())); + Value allocatedPtr = memref.allocatedPtr(rewriter, op.getLoc()); + Value casted = allocatedPtr; + if (!getTypeConverter()->useOpaquePointers()) + casted = rewriter.create(op.getLoc(), getVoidPtrType(), + allocatedPtr); + rewriter.replaceOpWithNewOp(op, freeFunc, casted); return success(); } @@ -421,25 +431,32 @@ struct DimOpLowering : public ConvertOpToLLVMPattern { // operations. UnrankedMemRefDescriptor unrankedDesc(adaptor.getSource()); Value underlyingRankedDesc = unrankedDesc.memRefDescPtr(rewriter, loc); - Value scalarMemRefDescPtr = rewriter.create( - loc, - LLVM::LLVMPointerType::get(typeConverter->convertType(scalarMemRefType), - addressSpace), - underlyingRankedDesc); + + Type elementType = typeConverter->convertType(scalarMemRefType); + Value scalarMemRefDescPtr; + if (getTypeConverter()->useOpaquePointers()) + scalarMemRefDescPtr = underlyingRankedDesc; + else + scalarMemRefDescPtr = rewriter.create( + loc, LLVM::LLVMPointerType::get(elementType, addressSpace), + underlyingRankedDesc); // Get pointer to offset field of memref descriptor. - Type indexPtrTy = LLVM::LLVMPointerType::get( + Type indexPtrTy = getTypeConverter()->getPointerType( getTypeConverter()->getIndexType(), addressSpace); Value offsetPtr = rewriter.create( - loc, indexPtrTy, scalarMemRefDescPtr, ArrayRef{0, 2}); + loc, indexPtrTy, elementType, scalarMemRefDescPtr, + ArrayRef{0, 2}); // The size value that we have to extract can be obtained using GEPop with // `dimOp.index() + 1` index argument. Value idxPlusOne = rewriter.create( loc, createIndexConstant(rewriter, loc, 1), adaptor.getIndex()); - Value sizePtr = - rewriter.create(loc, indexPtrTy, offsetPtr, idxPlusOne); - return rewriter.create(loc, sizePtr); + Value sizePtr = rewriter.create( + loc, indexPtrTy, getTypeConverter()->getIndexType(), offsetPtr, + idxPlusOne); + return rewriter.create( + loc, getTypeConverter()->getIndexType(), sizePtr); } std::optional getConstantDimIndex(memref::DimOp dimOp) const { @@ -549,7 +566,8 @@ struct GenericAtomicRMWOpLowering auto memRefType = atomicOp.getMemref().getType().cast(); auto dataPtr = getStridedElementPtr(loc, memRefType, adaptor.getMemref(), adaptor.getIndices(), rewriter); - Value init = rewriter.create(loc, dataPtr); + Value init = rewriter.create( + loc, typeConverter->convertType(memRefType.getElementType()), dataPtr); rewriter.create(loc, init, loopBlock); // Prepare the body of the loop block. @@ -570,12 +588,8 @@ struct GenericAtomicRMWOpLowering // Append the cmpxchg op to the end of the loop block. auto successOrdering = LLVM::AtomicOrdering::acq_rel; auto failureOrdering = LLVM::AtomicOrdering::monotonic; - auto boolType = IntegerType::get(rewriter.getContext(), 1); - auto pairType = LLVM::LLVMStructType::getLiteral(rewriter.getContext(), - {valueType, boolType}); auto cmpxchg = rewriter.create( - loc, pairType, dataPtr, loopArgument, result, successOrdering, - failureOrdering); + loc, dataPtr, loopArgument, result, successOrdering, failureOrdering); // Extract the %new_loaded and %ok values from the pair. Value newLoaded = rewriter.create(loc, cmpxchg, 0); Value ok = rewriter.create(loc, cmpxchg, 1); @@ -690,17 +704,18 @@ struct GetGlobalMemrefOpLowering : public AllocLikeOpLLVMLowering { unsigned memSpace = type.getMemorySpaceAsInt(); Type arrayTy = convertGlobalMemrefTypeToLLVM(type, *getTypeConverter()); - auto addressOf = rewriter.create( - loc, LLVM::LLVMPointerType::get(arrayTy, memSpace), - getGlobalOp.getName()); + Type resTy = getTypeConverter()->getPointerType(arrayTy, memSpace); + auto addressOf = + rewriter.create(loc, resTy, getGlobalOp.getName()); // Get the address of the first element in the array by creating a GEP with // the address of the GV as the base, and (rank + 1) number of 0 indices. Type elementType = typeConverter->convertType(type.getElementType()); - Type elementPtrType = LLVM::LLVMPointerType::get(elementType, memSpace); + Type elementPtrType = + getTypeConverter()->getPointerType(elementType, memSpace); auto gep = rewriter.create( - loc, elementPtrType, addressOf, + loc, elementPtrType, arrayTy, addressOf, SmallVector(type.getRank() + 1, 0)); // We do not expect the memref obtained using `memref.get_global` to be @@ -731,7 +746,9 @@ struct LoadOpLowering : public LoadStoreOpLowering { Value dataPtr = getStridedElementPtr(loadOp.getLoc(), type, adaptor.getMemref(), adaptor.getIndices(), rewriter); - rewriter.replaceOpWithNewOp(loadOp, dataPtr); + rewriter.replaceOpWithNewOp( + loadOp, typeConverter->convertType(type.getElementType()), dataPtr, 0, + false, loadOp.getNontemporal()); return success(); } }; @@ -748,7 +765,8 @@ struct StoreOpLowering : public LoadStoreOpLowering { Value dataPtr = getStridedElementPtr(op.getLoc(), type, adaptor.getMemref(), adaptor.getIndices(), rewriter); - rewriter.replaceOpWithNewOp(op, adaptor.getValue(), dataPtr); + rewriter.replaceOpWithNewOp(op, adaptor.getValue(), dataPtr, + 0, false, op.getNontemporal()); return success(); } }; @@ -852,10 +870,14 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern { // ptr = AllocaOp sizeof(MemRefDescriptor) auto ptr = getTypeConverter()->promoteOneMemRefDescriptor( loc, adaptor.getSource(), rewriter); + // voidptr = BitCastOp srcType* to void* - auto voidPtr = - rewriter.create(loc, getVoidPtrType(), ptr) - .getResult(); + Value voidPtr; + if (getTypeConverter()->useOpaquePointers()) + voidPtr = ptr; + else + voidPtr = rewriter.create(loc, getVoidPtrType(), ptr); + // rank = ConstantOp srcRank auto rankVal = rewriter.create( loc, getIndexType(), rewriter.getIndexAttr(rank)); @@ -876,13 +898,16 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern { // ptr = ExtractValueOp src, 1 auto ptr = memRefDesc.memRefDescPtr(rewriter, loc); // castPtr = BitCastOp i8* to structTy* - auto castPtr = - rewriter - .create( - loc, LLVM::LLVMPointerType::get(targetStructType), ptr) - .getResult(); + Value castPtr; + if (getTypeConverter()->useOpaquePointers()) + castPtr = ptr; + else + castPtr = rewriter.create( + loc, LLVM::LLVMPointerType::get(targetStructType), ptr); + // struct = LoadOp castPtr - auto loadOp = rewriter.create(loc, castPtr); + auto loadOp = + rewriter.create(loc, targetStructType, castPtr); rewriter.replaceOp(memRefCastOp, loadOp.getResult()); } else { llvm_unreachable("Unsupported unranked memref to unranked memref cast"); @@ -920,15 +945,17 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern { Value totalSize = rewriter.create(loc, numElements, sizeInBytes); + Type elementType = typeConverter->convertType(srcType.getElementType()); + Value srcBasePtr = srcDesc.alignedPtr(rewriter, loc); Value srcOffset = srcDesc.offset(rewriter, loc); - Value srcPtr = rewriter.create(loc, srcBasePtr.getType(), - srcBasePtr, srcOffset); + Value srcPtr = rewriter.create( + loc, srcBasePtr.getType(), elementType, srcBasePtr, srcOffset); MemRefDescriptor targetDesc(adaptor.getTarget()); Value targetBasePtr = targetDesc.alignedPtr(rewriter, loc); Value targetOffset = targetDesc.offset(rewriter, loc); - Value targetPtr = rewriter.create(loc, targetBasePtr.getType(), - targetBasePtr, targetOffset); + Value targetPtr = rewriter.create( + loc, targetBasePtr.getType(), elementType, targetBasePtr, targetOffset); Value isVolatile = rewriter.create(loc, rewriter.getBoolAttr(false)); rewriter.create(loc, targetPtr, srcPtr, totalSize, @@ -952,9 +979,13 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern { auto *typeConverter = getTypeConverter(); auto ptr = typeConverter->promoteOneMemRefDescriptor(loc, ranked, rewriter); - auto voidPtr = - rewriter.create(loc, getVoidPtrType(), ptr) - .getResult(); + + Value voidPtr; + if (getTypeConverter()->useOpaquePointers()) + voidPtr = ptr; + else + voidPtr = rewriter.create(loc, getVoidPtrType(), ptr); + auto unrankedType = UnrankedMemRefType::get(type.getElementType(), type.getMemorySpace()); return UnrankedMemRefDescriptor::pack(rewriter, loc, *typeConverter, @@ -977,9 +1008,9 @@ struct MemRefCopyOpLowering : public ConvertOpToLLVMPattern { auto one = rewriter.create(loc, getIndexType(), rewriter.getIndexAttr(1)); auto promote = [&](Value desc) { - auto ptrType = LLVM::LLVMPointerType::get(desc.getType()); + Type ptrType = getTypeConverter()->getPointerType(desc.getType()); auto allocated = - rewriter.create(loc, ptrType, ValueRange{one}); + rewriter.create(loc, ptrType, desc.getType(), one); rewriter.create(loc, desc, allocated); return allocated; }; @@ -1070,8 +1101,8 @@ static void extractPointersAndOffset(Location loc, operandType.cast().getMemorySpaceAsInt(); Type elementType = operandType.cast().getElementType(); Type llvmElementType = typeConverter.convertType(elementType); - Type elementPtrPtrType = LLVM::LLVMPointerType::get( - LLVM::LLVMPointerType::get(llvmElementType, memorySpace)); + LLVM::LLVMPointerType elementPtrType = + typeConverter.getPointerType(llvmElementType, memorySpace); // Extract pointer to the underlying ranked memref descriptor and cast it to // ElemType**. @@ -1079,12 +1110,12 @@ static void extractPointersAndOffset(Location loc, Value underlyingDescPtr = unrankedDesc.memRefDescPtr(rewriter, loc); *allocatedPtr = UnrankedMemRefDescriptor::allocatedPtr( - rewriter, loc, underlyingDescPtr, elementPtrPtrType); + rewriter, loc, underlyingDescPtr, elementPtrType); *alignedPtr = UnrankedMemRefDescriptor::alignedPtr( - rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType); + rewriter, loc, typeConverter, underlyingDescPtr, elementPtrType); if (offset != nullptr) { *offset = UnrankedMemRefDescriptor::offset( - rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType); + rewriter, loc, typeConverter, underlyingDescPtr, elementPtrType); } } @@ -1279,7 +1310,8 @@ struct MemRefReshapeOpLowering UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(), targetDesc, sizes); Value underlyingDescPtr = rewriter.create( - loc, getVoidPtrType(), sizes.front(), std::nullopt); + loc, getVoidPtrType(), IntegerType::get(getContext(), 8), + sizes.front()); targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr); // Extract pointers and offset from the source memref. @@ -1290,22 +1322,22 @@ struct MemRefReshapeOpLowering // Set pointers and offset. Type llvmElementType = typeConverter->convertType(elementType); - auto elementPtrPtrType = LLVM::LLVMPointerType::get( - LLVM::LLVMPointerType::get(llvmElementType, addressSpace)); + LLVM::LLVMPointerType elementPtrType = + getTypeConverter()->getPointerType(llvmElementType, addressSpace); + UnrankedMemRefDescriptor::setAllocatedPtr(rewriter, loc, underlyingDescPtr, - elementPtrPtrType, allocatedPtr); + elementPtrType, allocatedPtr); UnrankedMemRefDescriptor::setAlignedPtr(rewriter, loc, *getTypeConverter(), - underlyingDescPtr, - elementPtrPtrType, alignedPtr); + underlyingDescPtr, elementPtrType, + alignedPtr); UnrankedMemRefDescriptor::setOffset(rewriter, loc, *getTypeConverter(), - underlyingDescPtr, elementPtrPtrType, + underlyingDescPtr, elementPtrType, offset); // Use the offset pointer as base for further addressing. Copy over the new // shape and compute strides. For this, we create a loop from rank-1 to 0. Value targetSizesBase = UnrankedMemRefDescriptor::sizeBasePtr( - rewriter, loc, *getTypeConverter(), underlyingDescPtr, - elementPtrPtrType); + rewriter, loc, *getTypeConverter(), underlyingDescPtr, elementPtrType); Value targetStridesBase = UnrankedMemRefDescriptor::strideBasePtr( rewriter, loc, *getTypeConverter(), targetSizesBase, resultRank); Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc); @@ -1341,10 +1373,12 @@ struct MemRefReshapeOpLowering rewriter.setInsertionPointToStart(bodyBlock); // Copy size from shape to descriptor. - Type llvmIndexPtrType = LLVM::LLVMPointerType::get(indexType); - Value sizeLoadGep = rewriter.create(loc, llvmIndexPtrType, - shapeOperandPtr, indexArg); - Value size = rewriter.create(loc, sizeLoadGep); + Type llvmIndexPtrType = getTypeConverter()->getPointerType(indexType); + Value sizeLoadGep = rewriter.create( + loc, llvmIndexPtrType, + typeConverter->convertType(shapeMemRefType.getElementType()), + shapeOperandPtr, indexArg); + Value size = rewriter.create(loc, indexType, sizeLoadGep); UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(), targetSizesBase, indexArg, size); @@ -1530,22 +1564,35 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern { // Field 1: Copy the allocated pointer, used for malloc/free. Value allocatedPtr = sourceMemRef.allocatedPtr(rewriter, loc); auto srcMemRefType = viewOp.getSource().getType().cast(); - Value bitcastPtr = rewriter.create( - loc, - LLVM::LLVMPointerType::get(targetElementTy, - srcMemRefType.getMemorySpaceAsInt()), - allocatedPtr); + Value bitcastPtr; + if (getTypeConverter()->useOpaquePointers()) + bitcastPtr = allocatedPtr; + else + bitcastPtr = rewriter.create( + loc, + LLVM::LLVMPointerType::get(targetElementTy, + srcMemRefType.getMemorySpaceAsInt()), + allocatedPtr); + targetMemRef.setAllocatedPtr(rewriter, loc, bitcastPtr); // Field 2: Copy the actual aligned pointer to payload. Value alignedPtr = sourceMemRef.alignedPtr(rewriter, loc); alignedPtr = rewriter.create( - loc, alignedPtr.getType(), alignedPtr, adaptor.getByteShift()); - bitcastPtr = rewriter.create( - loc, - LLVM::LLVMPointerType::get(targetElementTy, - srcMemRefType.getMemorySpaceAsInt()), - alignedPtr); + loc, alignedPtr.getType(), + typeConverter->convertType(srcMemRefType.getElementType()), alignedPtr, + adaptor.getByteShift()); + + if (getTypeConverter()->useOpaquePointers()) { + bitcastPtr = alignedPtr; + } else { + bitcastPtr = rewriter.create( + loc, + LLVM::LLVMPointerType::get(targetElementTy, + srcMemRefType.getMemorySpaceAsInt()), + alignedPtr); + } + targetMemRef.setAlignedPtr(rewriter, loc, bitcastPtr); // Field 3: The offset in the resulting type must be 0. This is because of @@ -1620,13 +1667,12 @@ struct AtomicRMWOpLowering : public LoadStoreOpLowering { auto maybeKind = matchSimpleAtomicOp(atomicOp); if (!maybeKind) return failure(); - auto resultType = adaptor.getValue().getType(); auto memRefType = atomicOp.getMemRefType(); auto dataPtr = getStridedElementPtr(atomicOp.getLoc(), memRefType, adaptor.getMemref(), adaptor.getIndices(), rewriter); rewriter.replaceOpWithNewOp( - atomicOp, resultType, *maybeKind, dataPtr, adaptor.getValue(), + atomicOp, *maybeKind, dataPtr, adaptor.getValue(), LLVM::AtomicOrdering::acq_rel); return success(); } @@ -1703,8 +1749,8 @@ class ExtractStridedMetadataOpLowering } // namespace -void mlir::populateMemRefToLLVMConversionPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns) { +void mlir::populateFinalizeMemRefToLLVMConversionPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns) { // clang-format off patterns.add< AllocaOpLowering, @@ -1741,9 +1787,11 @@ void mlir::populateMemRefToLLVMConversionPatterns(LLVMTypeConverter &converter, } namespace { -struct MemRefToLLVMConversionPass - : public impl::MemRefToLLVMConversionPassBase { - using MemRefToLLVMConversionPassBase::MemRefToLLVMConversionPassBase; +struct FinalizeMemRefToLLVMConversionPass + : public impl::FinalizeMemRefToLLVMConversionPassBase< + FinalizeMemRefToLLVMConversionPass> { + using FinalizeMemRefToLLVMConversionPassBase:: + FinalizeMemRefToLLVMConversionPassBase; void runOnOperation() override { Operation *op = getOperation(); @@ -1755,6 +1803,7 @@ struct MemRefToLLVMConversionPass : LowerToLLVMOptions::AllocLowering::Malloc); options.useGenericFunctions = useGenericFunctions; + options.useOpaquePointers = useOpaquePointers; if (indexBitwidth != kDeriveIndexBitwidthFromDataLayout) options.overrideIndexBitwidth(indexBitwidth); @@ -1762,7 +1811,7 @@ struct MemRefToLLVMConversionPass LLVMTypeConverter typeConverter(&getContext(), options, &dataLayoutAnalysis); RewritePatternSet patterns(&getContext()); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); LLVMConversionTarget target(getContext()); target.addLegalOp(); if (failed(applyPartialConversion(op, target, std::move(patterns)))) diff --git a/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp b/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp index 54f3e44f0c17..16863206ac05 100644 --- a/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp +++ b/mlir/lib/Conversion/OpenACCToLLVM/OpenACCToLLVM.cpp @@ -15,7 +15,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTOPENACCTOLLVM +#define GEN_PASS_DEF_CONVERTOPENACCTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -154,7 +154,9 @@ void mlir::populateOpenACCToLLVMConversionPatterns( namespace { struct ConvertOpenACCToLLVMPass - : public impl::ConvertOpenACCToLLVMBase { + : public impl::ConvertOpenACCToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -238,8 +240,3 @@ void ConvertOpenACCToLLVMPass::runOnOperation() { if (failed(applyPartialConversion(op, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> -mlir::createConvertOpenACCToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp index 6a2d375c69f7..249e21e90803 100644 --- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp +++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp @@ -20,7 +20,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTOPENMPTOLLVM +#define GEN_PASS_DEF_CONVERTOPENMPTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -137,7 +137,9 @@ void mlir::populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter, namespace { struct ConvertOpenMPToLLVMPass - : public impl::ConvertOpenMPToLLVMBase { + : public impl::ConvertOpenMPToLLVMPassBase { + using Base::Base; + void runOnOperation() override; }; } // namespace @@ -150,7 +152,7 @@ void ConvertOpenMPToLLVMPass::runOnOperation() { LLVMTypeConverter converter(&getContext()); arith::populateArithToLLVMConversionPatterns(converter, patterns); cf::populateControlFlowToLLVMConversionPatterns(converter, patterns); - populateMemRefToLLVMConversionPatterns(converter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(converter, patterns); populateFuncToLLVMConversionPatterns(converter, patterns); populateOpenMPToLLVMConversionPatterns(converter, patterns); @@ -161,7 +163,3 @@ void ConvertOpenMPToLLVMPass::runOnOperation() { if (failed(applyPartialConversion(module, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> mlir::createConvertOpenMPToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp index 1c3e35d4a91d..a4acf0431b72 100644 --- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp +++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp @@ -229,7 +229,7 @@ static omp::ReductionDeclareOp addAtomicRMW(OpBuilder &builder, builder.setInsertionPointToEnd(atomicBlock); Value loaded = builder.create(reduce.getLoc(), atomicBlock->getArgument(1)); - builder.create(reduce.getLoc(), type, atomicKind, + builder.create(reduce.getLoc(), atomicKind, atomicBlock->getArgument(0), loaded, LLVM::AtomicOrdering::monotonic); builder.create(reduce.getLoc(), ArrayRef()); diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp index 08512e9f31e9..7803593d73a7 100644 --- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp +++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp @@ -33,7 +33,7 @@ #include "llvm/Support/FormatVariadic.h" namespace mlir { -#define GEN_PASS_DEF_LOWERHOSTCODETOLLVM +#define GEN_PASS_DEF_LOWERHOSTCODETOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -281,8 +281,11 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern { }; class LowerHostCodeToLLVM - : public impl::LowerHostCodeToLLVMBase { + : public impl::LowerHostCodeToLLVMPassBase { public: + + using Base::Base; + void runOnOperation() override { ModuleOp module = getOperation(); @@ -303,7 +306,7 @@ class LowerHostCodeToLLVM RewritePatternSet patterns(context); LLVMTypeConverter typeConverter(context, options); mlir::arith::populateArithToLLVMConversionPatterns(typeConverter, patterns); - populateMemRefToLLVMConversionPatterns(typeConverter, patterns); + populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns); populateFuncToLLVMConversionPatterns(typeConverter, patterns); patterns.add(typeConverter); @@ -327,8 +330,3 @@ class LowerHostCodeToLLVM } }; } // namespace - -std::unique_ptr> -mlir::createLowerHostCodeToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp index a2f3c8adf395..7766d8d9a0c9 100644 --- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp +++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.cpp @@ -19,7 +19,7 @@ #include "mlir/Pass/Pass.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTSPIRVTOLLVM +#define GEN_PASS_DEF_CONVERTSPIRVTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -28,8 +28,11 @@ using namespace mlir; namespace { /// A pass converting MLIR SPIR-V operations into LLVM dialect. class ConvertSPIRVToLLVMPass - : public impl::ConvertSPIRVToLLVMBase { + : public impl::ConvertSPIRVToLLVMPassBase { void runOnOperation() override; + +public: + using Base::Base; }; } // namespace @@ -58,7 +61,3 @@ void ConvertSPIRVToLLVMPass::runOnOperation() { if (failed(applyPartialConversion(module, target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> mlir::createConvertSPIRVToLLVMPass() { - return std::make_unique(); -} diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp index a1667ff25b12..dcf04b7ed58d 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp @@ -26,7 +26,7 @@ #include "mlir/Transforms/GreedyPatternRewriteDriver.h" namespace mlir { -#define GEN_PASS_DEF_CONVERTVECTORTOLLVM +#define GEN_PASS_DEF_CONVERTVECTORTOLLVMPASS #include "mlir/Conversion/Passes.h.inc" } // namespace mlir @@ -35,15 +35,10 @@ using namespace mlir::vector; namespace { struct LowerVectorToLLVMPass - : public impl::ConvertVectorToLLVMBase { - LowerVectorToLLVMPass(const LowerVectorToLLVMOptions &options) { - this->reassociateFPReductions = options.reassociateFPReductions; - this->force32BitVectorIndices = options.force32BitVectorIndices; - this->armNeon = options.armNeon; - this->armSVE = options.armSVE; - this->amx = options.amx; - this->x86Vector = options.x86Vector; - } + : public impl::ConvertVectorToLLVMPassBase { + + using Base::Base; + // Override explicitly to allow conditional dialect dependence. void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); @@ -116,8 +111,3 @@ void LowerVectorToLLVMPass::runOnOperation() { applyPartialConversion(getOperation(), target, std::move(patterns)))) signalPassFailure(); } - -std::unique_ptr> -mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) { - return std::make_unique(options); -} diff --git a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp index ff590d4e7070..b38f849c186a 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/FunctionCallUtils.cpp @@ -79,11 +79,25 @@ LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintF64Fn(ModuleOp moduleOp) { LLVM::LLVMVoidType::get(moduleOp->getContext())); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintStrFn(ModuleOp moduleOp) { - return lookupOrCreateFn( - moduleOp, kPrintStr, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8)), - LLVM::LLVMVoidType::get(moduleOp->getContext())); +static LLVM::LLVMPointerType getCharPtr(MLIRContext *context, + bool opaquePointers) { + if (opaquePointers) + return LLVM::LLVMPointerType::get(context); + + return LLVM::LLVMPointerType::get(IntegerType::get(context, 8)); +} + +static LLVM::LLVMPointerType getVoidPtr(MLIRContext *context, + bool opaquePointers) { + // A char pointer and void ptr are the same in LLVM IR. + return getCharPtr(context, opaquePointers); +} + +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintStrFn(ModuleOp moduleOp, + bool opaquePointers) { + return lookupOrCreateFn(moduleOp, kPrintStr, + getCharPtr(moduleOp->getContext(), opaquePointers), + LLVM::LLVMVoidType::get(moduleOp->getContext())); } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintOpenFn(ModuleOp moduleOp) { @@ -107,45 +121,48 @@ LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreatePrintNewlineFn(ModuleOp moduleOp) { } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateMallocFn(ModuleOp moduleOp, - Type indexType) { + Type indexType, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kMalloc, indexType, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateAlignedAllocFn(ModuleOp moduleOp, - Type indexType) { + Type indexType, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kAlignedAlloc, {indexType, indexType}, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateFreeFn(ModuleOp moduleOp, + bool opaquePointers) { return LLVM::lookupOrCreateFn( - moduleOp, kFree, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8)), + moduleOp, kFree, getVoidPtr(moduleOp->getContext(), opaquePointers), LLVM::LLVMVoidType::get(moduleOp->getContext())); } LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAllocFn(ModuleOp moduleOp, - Type indexType) { + Type indexType, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kGenericAlloc, indexType, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } -LLVM::LLVMFuncOp -mlir::LLVM::lookupOrCreateGenericAlignedAllocFn(ModuleOp moduleOp, - Type indexType) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericAlignedAllocFn( + ModuleOp moduleOp, Type indexType, bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kGenericAlignedAlloc, {indexType, indexType}, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8))); + getVoidPtr(moduleOp->getContext(), opaquePointers)); } -LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(ModuleOp moduleOp) { +LLVM::LLVMFuncOp mlir::LLVM::lookupOrCreateGenericFreeFn(ModuleOp moduleOp, + bool opaquePointers) { return LLVM::lookupOrCreateFn( moduleOp, kGenericFree, - LLVM::LLVMPointerType::get(IntegerType::get(moduleOp->getContext(), 8)), + getVoidPtr(moduleOp->getContext(), opaquePointers), LLVM::LLVMVoidType::get(moduleOp->getContext())); } diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp index 3b6b67a137a9..808428937988 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp @@ -42,10 +42,10 @@ void LLVMDialect::registerAttributes() { //===----------------------------------------------------------------------===// bool DINodeAttr::classof(Attribute attr) { - return llvm::isa(attr); } @@ -63,7 +63,7 @@ bool DIScopeAttr::classof(Attribute attr) { //===----------------------------------------------------------------------===// bool DILocalScopeAttr::classof(Attribute attr) { - return llvm::isa(attr); + return llvm::isa(attr); } //===----------------------------------------------------------------------===// @@ -71,197 +71,10 @@ bool DILocalScopeAttr::classof(Attribute attr) { //===----------------------------------------------------------------------===// bool DITypeAttr::classof(Attribute attr) { - return llvm::isa(attr); } -//===----------------------------------------------------------------------===// -// DISubroutineTypeAttr -//===----------------------------------------------------------------------===// - -LogicalResult -DISubroutineTypeAttr::verify(function_ref emitError, - unsigned int callingConventions, - ArrayRef types) { - ArrayRef argumentTypes = - types.empty() ? types : types.drop_front(); - if (llvm::any_of(argumentTypes, [](DITypeAttr type) { - return type.isa(); - })) - return emitError() << "expected subroutine to have non-void argument types"; - return success(); -} - -//===----------------------------------------------------------------------===// -// LoopOptionsAttrBuilder -//===----------------------------------------------------------------------===// - -LoopOptionsAttrBuilder::LoopOptionsAttrBuilder(LoopOptionsAttr attr) - : options(attr.getOptions().begin(), attr.getOptions().end()) {} - -template -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setOption(LoopOptionCase tag, std::optional value) { - auto option = llvm::find_if( - options, [tag](auto option) { return option.first == tag; }); - if (option != options.end()) { - if (value) - option->second = *value; - else - options.erase(option); - } else { - options.push_back(LoopOptionsAttr::OptionValuePair(tag, *value)); - } - return *this; -} - -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableLICM(std::optional value) { - return setOption(LoopOptionCase::disable_licm, value); -} - -/// Set the `interleave_count` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setInterleaveCount(std::optional count) { - return setOption(LoopOptionCase::interleave_count, count); -} - -/// Set the `disable_unroll` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisableUnroll(std::optional value) { - return setOption(LoopOptionCase::disable_unroll, value); -} - -/// Set the `disable_pipeline` option to the provided value. If no value -/// is provided the option is deleted. -LoopOptionsAttrBuilder & -LoopOptionsAttrBuilder::setDisablePipeline(std::optional value) { - return setOption(LoopOptionCase::disable_pipeline, value); -} - -/// Set the `pipeline_initiation_interval` option to the provided value. -/// If no value is provided the option is deleted. -LoopOptionsAttrBuilder &LoopOptionsAttrBuilder::setPipelineInitiationInterval( - std::optional count) { - return setOption(LoopOptionCase::pipeline_initiation_interval, count); -} - -//===----------------------------------------------------------------------===// -// LoopOptionsAttr -//===----------------------------------------------------------------------===// - -template -static std::optional -getOption(ArrayRef> options, - LoopOptionCase option) { - auto it = - lower_bound(options, option, [](auto optionPair, LoopOptionCase option) { - return optionPair.first < option; - }); - if (it == options.end()) - return {}; - return static_cast(it->second); -} - -std::optional LoopOptionsAttr::disableUnroll() { - return getOption(getOptions(), LoopOptionCase::disable_unroll); -} - -std::optional LoopOptionsAttr::disableLICM() { - return getOption(getOptions(), LoopOptionCase::disable_licm); -} - -std::optional LoopOptionsAttr::interleaveCount() { - return getOption(getOptions(), LoopOptionCase::interleave_count); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get( - MLIRContext *context, - ArrayRef> sortedOptions) { - assert(llvm::is_sorted(sortedOptions, llvm::less_first()) && - "LoopOptionsAttr ctor expects a sorted options array"); - return Base::get(context, sortedOptions); -} - -/// Build the LoopOptions Attribute from a sorted array of individual options. -LoopOptionsAttr LoopOptionsAttr::get(MLIRContext *context, - LoopOptionsAttrBuilder &optionBuilders) { - llvm::sort(optionBuilders.options, llvm::less_first()); - return Base::get(context, optionBuilders.options); -} - -void LoopOptionsAttr::print(AsmPrinter &printer) const { - printer << "<"; - llvm::interleaveComma(getOptions(), printer, [&](auto option) { - printer << stringifyEnum(option.first) << " = "; - switch (option.first) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - printer << (option.second ? "true" : "false"); - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - printer << option.second; - break; - } - }); - printer << ">"; -} - -Attribute LoopOptionsAttr::parse(AsmParser &parser, Type type) { - if (failed(parser.parseLess())) - return {}; - - SmallVector> options; - llvm::SmallDenseSet seenOptions; - auto parseLoopOptions = [&]() -> ParseResult { - StringRef optionName; - if (parser.parseKeyword(&optionName)) - return failure(); - - auto option = symbolizeLoopOptionCase(optionName); - if (!option) - return parser.emitError(parser.getNameLoc(), "unknown loop option: ") - << optionName; - if (!seenOptions.insert(*option).second) - return parser.emitError(parser.getNameLoc(), "loop option present twice"); - if (failed(parser.parseEqual())) - return failure(); - - int64_t value; - switch (*option) { - case LoopOptionCase::disable_licm: - case LoopOptionCase::disable_unroll: - case LoopOptionCase::disable_pipeline: - if (succeeded(parser.parseOptionalKeyword("true"))) - value = 1; - else if (succeeded(parser.parseOptionalKeyword("false"))) - value = 0; - else { - return parser.emitError(parser.getNameLoc(), - "expected boolean value 'true' or 'false'"); - } - break; - case LoopOptionCase::interleave_count: - case LoopOptionCase::pipeline_initiation_interval: - if (failed(parser.parseInteger(value))) - return parser.emitError(parser.getNameLoc(), "expected integer value"); - break; - } - options.push_back(std::make_pair(*option, value)); - return success(); - }; - if (parser.parseCommaSeparatedList(parseLoopOptions) || parser.parseGreater()) - return {}; - - llvm::sort(options, llvm::less_first()); - return get(parser.getContext(), options); -} - //===----------------------------------------------------------------------===// // MemoryEffectsAttr //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 4f301ff00c95..510abda7a676 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -44,8 +44,6 @@ using mlir::LLVM::linkage::getMaxEnumValForLinkage; #include "mlir/Dialect/LLVMIR/LLVMOpsDialect.cpp.inc" -static constexpr const char kVolatileAttrName[] = "volatile_"; -static constexpr const char kNonTemporalAttrName[] = "nontemporal"; static constexpr const char kElemTypeAttrName[] = "elem_type"; #include "mlir/Dialect/LLVMIR/LLVMOpsInterfaces.cpp.inc" @@ -266,11 +264,42 @@ LogicalResult AllocaOp::verify() { // LLVM::BrOp //===----------------------------------------------------------------------===// +/// Check if the `loopAttr` references correct symbols. +static LogicalResult verifyLoopAnnotationAttr(LoopAnnotationAttr loopAttr, + Operation *op) { + if (!loopAttr) + return success(); + // If the `llvm.loop` attribute is present, enforce the following structure, + // which the module translation can assume. + ArrayRef parallelAccesses = loopAttr.getParallelAccesses(); + if (parallelAccesses.empty()) + return success(); + for (SymbolRefAttr accessGroupRef : parallelAccesses) { + StringAttr metadataName = accessGroupRef.getRootReference(); + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference a metadata op"; + StringAttr accessGroupName = accessGroupRef.getLeafReference(); + Operation *accessGroupOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); + if (!accessGroupOp) + return op->emitOpError() << "expected '" << accessGroupRef + << "' to reference an access_group op"; + } + return success(); +} + SuccessorOperands BrOp::getSuccessorOperands(unsigned index) { assert(index == 0 && "invalid successor index"); return SuccessorOperands(getDestOperandsMutable()); } +LogicalResult BrOp::verify() { + return verifyLoopAnnotationAttr(getLoopAnnotationAttr(), *this); +} + //===----------------------------------------------------------------------===// // LLVM::CondBrOp //===----------------------------------------------------------------------===// @@ -281,6 +310,24 @@ SuccessorOperands CondBrOp::getSuccessorOperands(unsigned index) { : getFalseDestOperandsMutable()); } +LogicalResult CondBrOp::verify() { + return verifyLoopAnnotationAttr(getLoopAnnotationAttr(), *this); +} + +void CondBrOp::build(OpBuilder &builder, OperationState &result, + Value condition, Block *trueDest, ValueRange trueOperands, + Block *falseDest, ValueRange falseOperands, + std::optional> weights) { + ElementsAttr weightsAttr; + if (weights) + weightsAttr = + builder.getI32VectorAttr({static_cast(weights->first), + static_cast(weights->second)}); + + build(builder, result, condition, trueOperands, falseOperands, weightsAttr, + /*loop_annotation=*/{}, trueDest, falseDest); +} + //===----------------------------------------------------------------------===// // LLVM::SwitchOp //===----------------------------------------------------------------------===// @@ -665,56 +712,63 @@ Type LLVM::GEPOp::getSourceElementType() { } //===----------------------------------------------------------------------===// -// Builder, printer and parser for for LLVM::LoadOp. +// LoadOp //===----------------------------------------------------------------------===// -LogicalResult verifySymbolAttribute( - Operation *op, StringRef attributeName, +/// Verifies the given array attribute contains symbol references and checks the +/// referenced symbol types using the provided verification function. +LogicalResult verifyMemOpSymbolRefs( + Operation *op, StringRef name, ArrayAttr symbolRefs, llvm::function_ref verifySymbolType) { - if (Attribute attribute = op->getAttr(attributeName)) { - // Verify that the attribute is a symbol ref array attribute, - // because this constraint is not verified for all attribute - // names processed here (e.g. 'tbaa'). This verification - // is redundant in some cases. - if (!(attribute.isa() && - llvm::all_of(attribute.cast(), [&](Attribute attr) { - return attr && attr.isa(); - }))) - return op->emitOpError("attribute '") - << attributeName - << "' failed to satisfy constraint: symbol ref array attribute"; - - for (SymbolRefAttr symbolRef : - attribute.cast().getAsRange()) { - StringAttr metadataName = symbolRef.getRootReference(); - StringAttr symbolName = symbolRef.getLeafReference(); - // We want @metadata::@symbol, not just @symbol - if (metadataName == symbolName) { - return op->emitOpError() << "expected '" << symbolRef - << "' to specify a fully qualified reference"; - } - auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to reference a metadata op"; - Operation *symbolOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); - if (!symbolOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to be a valid reference"; - if (failed(verifySymbolType(symbolOp, symbolRef))) { - return failure(); - } + assert(symbolRefs && "expected a non-null attribute"); + + // Verify that the attribute is a symbol ref array attribute, + // because this constraint is not verified for all attribute + // names processed here (e.g. 'tbaa'). This verification + // is redundant in some cases. + if (!llvm::all_of(symbolRefs, [](Attribute attr) { + return attr && attr.isa(); + })) + return op->emitOpError("attribute '") + << name + << "' failed to satisfy constraint: symbol ref array attribute"; + + for (SymbolRefAttr symbolRef : symbolRefs.getAsRange()) { + StringAttr metadataName = symbolRef.getRootReference(); + StringAttr symbolName = symbolRef.getLeafReference(); + // We want @metadata::@symbol, not just @symbol + if (metadataName == symbolName) { + return op->emitOpError() << "expected '" << symbolRef + << "' to specify a fully qualified reference"; + } + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to reference a metadata op"; + Operation *symbolOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); + if (!symbolOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to be a valid reference"; + if (failed(verifySymbolType(symbolOp, symbolRef))) { + return failure(); } } + return success(); } -// Verifies that metadata ops are wired up properly. +/// Verifies the given array attribute contains symbol references that point to +/// metadata operations of the given type. template -static LogicalResult verifyOpMetadata(Operation *op, StringRef attributeName) { +static LogicalResult +verifyMemOpSymbolRefsPointTo(Operation *op, StringRef name, + std::optional symbolRefs) { + if (!symbolRefs) + return success(); + auto verifySymbolType = [op](Operation *symbolOp, SymbolRefAttr symbolRef) -> LogicalResult { if (!isa(symbolOp)) { @@ -724,59 +778,105 @@ static LogicalResult verifyOpMetadata(Operation *op, StringRef attributeName) { } return success(); }; - - return verifySymbolAttribute(op, attributeName, verifySymbolType); + return verifyMemOpSymbolRefs(op, name, *symbolRefs, verifySymbolType); } -static LogicalResult verifyMemoryOpMetadata(Operation *op) { - // access_groups - if (failed(verifyOpMetadata( - op, LLVMDialect::getAccessGroupsAttrName()))) +/// Verifies the types of the metadata operations referenced by aliasing and +/// access group metadata. +template +LogicalResult verifyMemOpMetadata(OpTy memOp) { + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getAccessGroupsAttrName(), memOp.getAccessGroups()))) return failure(); - // alias_scopes - if (failed(verifyOpMetadata( - op, LLVMDialect::getAliasScopesAttrName()))) + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getAliasScopesAttrName(), memOp.getAliasScopes()))) return failure(); - // noalias_scopes - if (failed(verifyOpMetadata( - op, LLVMDialect::getNoAliasScopesAttrName()))) + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getNoaliasScopesAttrName(), memOp.getNoaliasScopes()))) return failure(); - // tbaa - if (failed(verifyOpMetadata(op, - LLVMDialect::getTBAAAttrName()))) + if (failed(verifyMemOpSymbolRefsPointTo( + memOp, memOp.getTbaaAttrName(), memOp.getTbaa()))) return failure(); return success(); } -LogicalResult LoadOp::verify() { return verifyMemoryOpMetadata(*this); } +/// Returns true if the given type is supported by atomic operations. All +/// integer and float types with limited bit width are supported. Additionally, +/// depending on the operation pointers may be supported as well. +static bool isTypeCompatibleWithAtomicOp(Type type, bool isPointerTypeAllowed) { + if (type.isa()) + return isPointerTypeAllowed; -void LoadOp::build(OpBuilder &builder, OperationState &result, Type t, - Value addr, unsigned alignment, bool isVolatile, - bool isNonTemporal) { - result.addOperands(addr); - result.addTypes(t); - if (isVolatile) - result.addAttribute(kVolatileAttrName, builder.getUnitAttr()); - if (isNonTemporal) - result.addAttribute(kNonTemporalAttrName, builder.getUnitAttr()); - if (alignment != 0) - result.addAttribute("alignment", builder.getI64IntegerAttr(alignment)); + std::optional bitWidth = std::nullopt; + if (auto floatType = type.dyn_cast()) { + if (!isCompatibleFloatingPointType(type)) + return false; + bitWidth = floatType.getWidth(); + } + if (auto integerType = type.dyn_cast()) + bitWidth = integerType.getWidth(); + // The type is neither an integer, float, or pointer type. + if (!bitWidth) + return false; + return *bitWidth == 8 || *bitWidth == 16 || *bitWidth == 32 || + *bitWidth == 64; } -void LoadOp::print(OpAsmPrinter &p) { - p << ' '; - if (getVolatile_()) - p << "volatile "; - p << getAddr(); - p.printOptionalAttrDict((*this)->getAttrs(), - {kVolatileAttrName, kElemTypeAttrName}); - p << " : " << getAddr().getType(); - if (getAddr().getType().cast().isOpaque()) - p << " -> " << getType(); +/// Verifies the attributes and the type of atomic memory access operations. +template +LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType, + ArrayRef unsupportedOrderings) { + if (memOp.getOrdering() != AtomicOrdering::not_atomic) { + if (!isTypeCompatibleWithAtomicOp(valueType, + /*isPointerTypeAllowed=*/true)) + return memOp.emitOpError("unsupported type ") + << valueType << " for atomic access"; + if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering())) + return memOp.emitOpError("unsupported ordering '") + << stringifyAtomicOrdering(memOp.getOrdering()) << "'"; + if (!memOp.getAlignment()) + return memOp.emitOpError("expected alignment for atomic access"); + return success(); + } + if (memOp.getSyncscope()) + return memOp.emitOpError( + "expected syncscope to be null for non-atomic access"); + return success(); +} + +LogicalResult LoadOp::verify() { + Type valueType = getResult().getType(); + if (failed(verifyAtomicMemOp( + *this, valueType, + {AtomicOrdering::release, AtomicOrdering::acq_rel}))) + return failure(); + + return verifyMemOpMetadata(*this); +} + +void LoadOp::build(OpBuilder &builder, OperationState &state, Value addr, + unsigned alignment, bool isVolatile, bool isNonTemporal) { + auto type = addr.getType().cast().getElementType(); + assert(type && "must provide explicit element type to the constructor " + "when the pointer type is opaque"); + build(builder, state, type, addr, alignment, isVolatile, isNonTemporal); +} + +void LoadOp::build(OpBuilder &builder, OperationState &state, Type type, + Value addr, unsigned alignment, bool isVolatile, + bool isNonTemporal, AtomicOrdering ordering, + StringRef syncscope) { + build(builder, state, type, addr, + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, + /*tbaa=*/nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, + isNonTemporal, ordering, + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); } // Extract the pointee type from the LLVM pointer type wrapped in MLIR. Return @@ -792,103 +892,337 @@ getLoadStoreElementType(OpAsmParser &parser, Type type, SMLoc trailingTypeLoc) { return llvmTy.getElementType(); } -// ::= `llvm.load` `volatile` ssa-use attribute-dict? `:` type -// (`->` type)? -ParseResult LoadOp::parse(OpAsmParser &parser, OperationState &result) { - OpAsmParser::UnresolvedOperand addr; - Type type; +/// Parses the LoadOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static ParseResult parseLoadType(OpAsmParser &parser, Type &type, + Type &elementType) { SMLoc trailingTypeLoc; - - if (succeeded(parser.parseOptionalKeyword("volatile"))) - result.addAttribute(kVolatileAttrName, parser.getBuilder().getUnitAttr()); - - if (parser.parseOperand(addr) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type) || - parser.resolveOperand(addr, type, result.operands)) + if (parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type)) return failure(); - std::optional elemTy = + std::optional pointerElementType = getLoadStoreElementType(parser, type, trailingTypeLoc); - if (!elemTy) + if (!pointerElementType) return failure(); - if (*elemTy) { - result.addTypes(*elemTy); + if (*pointerElementType) { + elementType = *pointerElementType; return success(); } - Type trailingType; - if (parser.parseArrow() || parser.parseType(trailingType)) + if (parser.parseArrow() || parser.parseType(elementType)) return failure(); - result.addTypes(trailingType); return success(); } +/// Prints the LoadOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static void printLoadType(OpAsmPrinter &printer, Operation *op, Type type, + Type elementType) { + printer << type; + auto pointerType = cast(type); + if (pointerType.isOpaque()) + printer << " -> " << elementType; +} + //===----------------------------------------------------------------------===// -// Builder, printer and parser for LLVM::StoreOp. +// StoreOp //===----------------------------------------------------------------------===// -LogicalResult StoreOp::verify() { return verifyMemoryOpMetadata(*this); } +LogicalResult StoreOp::verify() { + Type valueType = getValue().getType(); + if (failed(verifyAtomicMemOp( + *this, valueType, + {AtomicOrdering::acquire, AtomicOrdering::acq_rel}))) + return failure(); -void StoreOp::build(OpBuilder &builder, OperationState &result, Value value, + return verifyMemOpMetadata(*this); +} + +void StoreOp::build(OpBuilder &builder, OperationState &state, Value value, Value addr, unsigned alignment, bool isVolatile, - bool isNonTemporal) { - result.addOperands({value, addr}); - result.addTypes({}); - if (isVolatile) - result.addAttribute(kVolatileAttrName, builder.getUnitAttr()); - if (isNonTemporal) - result.addAttribute(kNonTemporalAttrName, builder.getUnitAttr()); - if (alignment != 0) - result.addAttribute("alignment", builder.getI64IntegerAttr(alignment)); + bool isNonTemporal, AtomicOrdering ordering, + StringRef syncscope) { + build(builder, state, value, addr, /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, + isNonTemporal, ordering, + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); +} + +/// Parses the StoreOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static ParseResult parseStoreType(OpAsmParser &parser, Type &elementType, + Type &type) { + SMLoc trailingTypeLoc; + if (parser.getCurrentLocation(&trailingTypeLoc) || + parser.parseType(elementType)) + return failure(); + + if (succeeded(parser.parseOptionalComma())) + return parser.parseType(type); + + // Extract the element type from the pointer type. + type = elementType; + std::optional pointerElementType = + getLoadStoreElementType(parser, type, trailingTypeLoc); + if (!pointerElementType) + return failure(); + elementType = *pointerElementType; + return success(); +} + +/// Prints the StoreOp type either using the typed or opaque pointer format. +// TODO: Drop once the typed pointer assembly format is not needed anymore. +static void printStoreType(OpAsmPrinter &printer, Operation *op, + Type elementType, Type type) { + auto pointerType = cast(type); + if (pointerType.isOpaque()) + printer << elementType << ", "; + printer << type; +} + +//===----------------------------------------------------------------------===// +// CallOp +//===----------------------------------------------------------------------===// + +void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, + StringRef callee, ValueRange args) { + build(builder, state, results, builder.getStringAttr(callee), args); } -void StoreOp::print(OpAsmPrinter &p) { +void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, + StringAttr callee, ValueRange args) { + build(builder, state, results, SymbolRefAttr::get(callee), args, nullptr, + nullptr); +} + +void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, + FlatSymbolRefAttr callee, ValueRange args) { + build(builder, state, results, callee, args, nullptr, nullptr); +} + +void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func, + ValueRange args) { + SmallVector results; + Type resultType = func.getFunctionType().getReturnType(); + if (!resultType.isa()) + results.push_back(resultType); + build(builder, state, results, SymbolRefAttr::get(func), args, nullptr, + nullptr); +} + +CallInterfaceCallable CallOp::getCallableForCallee() { + // Direct call. + if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) + return calleeAttr; + // Indirect call, callee Value is the first operand. + return getOperand(0); +} + +Operation::operand_range CallOp::getArgOperands() { + return getOperands().drop_front(getCallee().has_value() ? 0 : 1); +} + +LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { + if (getNumResults() > 1) + return emitOpError("must have 0 or 1 result"); + + // Type for the callee, we'll get it differently depending if it is a direct + // or indirect call. + Type fnType; + + bool isIndirect = false; + + // If this is an indirect call, the callee attribute is missing. + FlatSymbolRefAttr calleeName = getCalleeAttr(); + if (!calleeName) { + isIndirect = true; + if (!getNumOperands()) + return emitOpError( + "must have either a `callee` attribute or at least an operand"); + auto ptrType = getOperand(0).getType().dyn_cast(); + if (!ptrType) + return emitOpError("indirect call expects a pointer as callee: ") + << getOperand(0).getType(); + + if (ptrType.isOpaque()) + return success(); + + fnType = ptrType.getElementType(); + } else { + Operation *callee = + symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr()); + if (!callee) + return emitOpError() + << "'" << calleeName.getValue() + << "' does not reference a symbol in the current scope"; + auto fn = dyn_cast(callee); + if (!fn) + return emitOpError() << "'" << calleeName.getValue() + << "' does not reference a valid LLVM function"; + + fnType = fn.getFunctionType(); + } + + LLVMFunctionType funcType = fnType.dyn_cast(); + if (!funcType) + return emitOpError("callee does not have a functional type: ") << fnType; + + // Indirect variadic function calls are not supported since the translation to + // LLVM IR reconstructs the LLVM function type from the argument and result + // types. An additional type attribute that stores the LLVM function type + // would be needed to distinguish normal and variadic function arguments. + // TODO: Support indirect calls to variadic function pointers. + if (isIndirect && funcType.isVarArg()) + return emitOpError() + << "indirect calls to variadic functions are not supported"; + + // Verify that the operand and result types match the callee. + + if (!funcType.isVarArg() && + funcType.getNumParams() != (getNumOperands() - isIndirect)) + return emitOpError() << "incorrect number of operands (" + << (getNumOperands() - isIndirect) + << ") for callee (expecting: " + << funcType.getNumParams() << ")"; + + if (funcType.getNumParams() > (getNumOperands() - isIndirect)) + return emitOpError() << "incorrect number of operands (" + << (getNumOperands() - isIndirect) + << ") for varargs callee (expecting at least: " + << funcType.getNumParams() << ")"; + + for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i) + if (getOperand(i + isIndirect).getType() != funcType.getParamType(i)) + return emitOpError() << "operand type mismatch for operand " << i << ": " + << getOperand(i + isIndirect).getType() + << " != " << funcType.getParamType(i); + + if (getNumResults() == 0 && + !funcType.getReturnType().isa()) + return emitOpError() << "expected function call to produce a value"; + + if (getNumResults() != 0 && + funcType.getReturnType().isa()) + return emitOpError() + << "calling function with void result must not produce values"; + + if (getNumResults() > 1) + return emitOpError() + << "expected LLVM function call to produce 0 or 1 result"; + + if (getNumResults() && getResult().getType() != funcType.getReturnType()) + return emitOpError() << "result type mismatch: " << getResult().getType() + << " != " << funcType.getReturnType(); + + return success(); +} + +void CallOp::print(OpAsmPrinter &p) { + auto callee = getCallee(); + bool isDirect = callee.has_value(); + + // Print the direct callee if present as a function attribute, or an indirect + // callee (first operand) otherwise. p << ' '; - if (getVolatile_()) - p << "volatile "; - p << getValue() << ", " << getAddr(); - p.printOptionalAttrDict((*this)->getAttrs(), {kVolatileAttrName}); + if (isDirect) + p.printSymbolName(callee.value()); + else + p << getOperand(0); + + auto args = getOperands().drop_front(isDirect ? 0 : 1); + p << '(' << args << ')'; + p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"callee"}); + p << " : "; - if (getAddr().getType().cast().isOpaque()) - p << getValue().getType() << ", "; - p << getAddr().getType(); + if (!isDirect) + p << getOperand(0).getType() << ", "; + + // Reconstruct the function MLIR function type from operand and result types. + p.printFunctionalType(args.getTypes(), getResultTypes()); } -// ::= `llvm.store` `volatile` ssa-use `,` ssa-use -// attribute-dict? `:` type (`,` type)? -ParseResult StoreOp::parse(OpAsmParser &parser, OperationState &result) { - OpAsmParser::UnresolvedOperand addr, value; - Type type; - SMLoc trailingTypeLoc; +/// Parses the type of a call operation and resolves the operands if the parsing +/// succeeds. Returns failure otherwise. +static ParseResult parseCallTypeAndResolveOperands( + OpAsmParser &parser, OperationState &result, bool isDirect, + ArrayRef operands) { + SMLoc trailingTypesLoc = parser.getCurrentLocation(); + SmallVector types; + if (parser.parseColonTypeList(types)) + return failure(); - if (succeeded(parser.parseOptionalKeyword("volatile"))) - result.addAttribute(kVolatileAttrName, parser.getBuilder().getUnitAttr()); + if (isDirect && types.size() != 1) + return parser.emitError(trailingTypesLoc, + "expected direct call to have 1 trailing type"); + if (!isDirect && types.size() != 2) + return parser.emitError(trailingTypesLoc, + "expected indirect call to have 2 trailing types"); - if (parser.parseOperand(value) || parser.parseComma() || - parser.parseOperand(addr) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type)) + auto funcType = types.pop_back_val().dyn_cast(); + if (!funcType) + return parser.emitError(trailingTypesLoc, + "expected trailing function type"); + if (funcType.getNumResults() > 1) + return parser.emitError(trailingTypesLoc, + "expected function with 0 or 1 result"); + if (funcType.getNumResults() == 1 && + funcType.getResult(0).isa()) + return parser.emitError(trailingTypesLoc, + "expected a non-void result type"); + + // The head element of the types list matches the callee type for + // indirect calls, while the types list is emtpy for direct calls. + // Append the function input types to resolve the call operation + // operands. + llvm::append_range(types, funcType.getInputs()); + if (parser.resolveOperands(operands, types, parser.getNameLoc(), + result.operands)) return failure(); + if (funcType.getNumResults() != 0) + result.addTypes(funcType.getResults()); - Type operandType; - if (succeeded(parser.parseOptionalComma())) { - operandType = type; - if (parser.parseType(type)) - return failure(); - } else { - std::optional maybeOperandType = - getLoadStoreElementType(parser, type, trailingTypeLoc); - if (!maybeOperandType) - return failure(); - operandType = *maybeOperandType; + return success(); +} + +/// Parses an optional function pointer operand before the call argument list +/// for indirect calls, or stops parsing at the function identifier otherwise. +static ParseResult parseOptionalCallFuncPtr( + OpAsmParser &parser, + SmallVectorImpl &operands) { + OpAsmParser::UnresolvedOperand funcPtrOperand; + OptionalParseResult parseResult = parser.parseOptionalOperand(funcPtrOperand); + if (parseResult.has_value()) { + if (failed(*parseResult)) + return *parseResult; + operands.push_back(funcPtrOperand); } + return success(); +} + +// ::= `llvm.call` (function-id | ssa-use)`(` ssa-use-list `)` +// attribute-dict? `:` (type `,`)? function-type +ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) { + SymbolRefAttr funcAttr; + SmallVector operands; - if (parser.resolveOperand(value, operandType, result.operands) || - parser.resolveOperand(addr, type, result.operands)) + // Parse a function pointer for indirect calls. + if (parseOptionalCallFuncPtr(parser, operands)) return failure(); + bool isDirect = operands.empty(); - return success(); + // Parse a function identifier for direct calls. + if (isDirect) + if (parser.parseAttribute(funcAttr, "callee", result.attributes)) + return failure(); + + // Parse the function arguments. + if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser.parseOptionalAttrDict(result.attributes)) + return failure(); + + // Parse the trailing type list and resolve the operands. + return parseCallTypeAndResolveOperands(parser, result, isDirect, operands); } ///===---------------------------------------------------------------------===// @@ -949,93 +1283,48 @@ void InvokeOp::print(OpAsmPrinter &p) { p.printOptionalAttrDict((*this)->getAttrs(), {InvokeOp::getOperandSegmentSizeAttr(), "callee"}); + p << " : "; + if (!isDirect) + p << getOperand(0).getType() << ", "; p.printFunctionalType(llvm::drop_begin(getOperandTypes(), isDirect ? 0 : 1), getResultTypes()); } -/// ::= `llvm.invoke` (function-id | ssa-use) `(` ssa-use-list `)` -/// `to` bb-id (`[` ssa-use-and-type-list `]`)? -/// `unwind` bb-id (`[` ssa-use-and-type-list `]`)? -/// attribute-dict? `:` function-type +// ::= `llvm.invoke` (function-id | ssa-use) +// `(` ssa-use-list `)` +// `to` bb-id (`[` ssa-use-and-type-list `]`)? +// `unwind` bb-id (`[` ssa-use-and-type-list `]`)? +// attribute-dict? `:` (type `,`)? function-type ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) { SmallVector operands; - FunctionType funcType; SymbolRefAttr funcAttr; - SMLoc trailingTypeLoc; Block *normalDest, *unwindDest; SmallVector normalOperands, unwindOperands; Builder &builder = parser.getBuilder(); - // Parse an operand list that will, in practice, contain 0 or 1 operand. In - // case of an indirect call, there will be 1 operand before `(`. In case of a - // direct call, there will be no operands and the parser will stop at the - // function identifier without complaining. - if (parser.parseOperandList(operands)) + // Parse a function pointer for indirect calls. + if (parseOptionalCallFuncPtr(parser, operands)) return failure(); bool isDirect = operands.empty(); - // Optionally parse a function identifier. + // Parse a function identifier for direct calls. if (isDirect && parser.parseAttribute(funcAttr, "callee", result.attributes)) return failure(); + // Parse the function arguments. if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) || parser.parseKeyword("to") || parser.parseSuccessorAndUseList(normalDest, normalOperands) || parser.parseKeyword("unwind") || parser.parseSuccessorAndUseList(unwindDest, unwindOperands) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(funcType)) + parser.parseOptionalAttrDict(result.attributes)) return failure(); - if (isDirect) { - // Make sure types match. - if (parser.resolveOperands(operands, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); - result.addTypes(funcType.getResults()); - } else { - // Construct the LLVM IR Dialect function type that the first operand - // should match. - if (funcType.getNumResults() > 1) - return parser.emitError(trailingTypeLoc, - "expected function with 0 or 1 result"); - - Type llvmResultType; - if (funcType.getNumResults() == 0) { - llvmResultType = LLVM::LLVMVoidType::get(builder.getContext()); - } else { - llvmResultType = funcType.getResult(0); - if (!isCompatibleType(llvmResultType)) - return parser.emitError(trailingTypeLoc, - "expected result to have LLVM type"); - } - - SmallVector argTypes; - argTypes.reserve(funcType.getNumInputs()); - for (Type ty : funcType.getInputs()) { - if (isCompatibleType(ty)) - argTypes.push_back(ty); - else - return parser.emitError(trailingTypeLoc, - "expected LLVM types as inputs"); - } - - auto llvmFuncType = LLVM::LLVMFunctionType::get(llvmResultType, argTypes); - auto wrappedFuncType = LLVM::LLVMPointerType::get(llvmFuncType); - - auto funcArguments = llvm::ArrayRef(operands).drop_front(); - - // Make sure that the first operand (indirect callee) matches the wrapped - // LLVM IR function type, and that the types of the other call operands - // match the types of the function arguments. - if (parser.resolveOperand(operands[0], wrappedFuncType, result.operands) || - parser.resolveOperands(funcArguments, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); + // Parse the trailing type list and resolve the function operands. + if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands)) + return failure(); - result.addTypes(llvmResultType); - } result.addSuccessors({normalDest, unwindDest}); result.addOperands(normalOperands); result.addOperands(unwindOperands); @@ -1079,291 +1368,60 @@ LogicalResult LandingpadOp::verify() { << "global addresses expected as operand to " "bitcast used in clauses for landingpad"; } - // NullOp and AddressOfOp allowed - if (value.getDefiningOp()) - continue; - if (value.getDefiningOp()) - continue; - return emitError("clause #") - << idx << " is not a known constant - null, addressof, bitcast"; - } - } - return success(); -} - -void LandingpadOp::print(OpAsmPrinter &p) { - p << (getCleanup() ? " cleanup " : " "); - - // Clauses - for (auto value : getOperands()) { - // Similar to llvm - if clause is an array type then it is filter - // clause else catch clause - bool isArrayTy = value.getType().isa(); - p << '(' << (isArrayTy ? "filter " : "catch ") << value << " : " - << value.getType() << ") "; - } - - p.printOptionalAttrDict((*this)->getAttrs(), {"cleanup"}); - - p << ": " << getType(); -} - -/// ::= `llvm.landingpad` `cleanup`? -/// ((`catch` | `filter`) operand-type ssa-use)* attribute-dict? -ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) { - // Check for cleanup - if (succeeded(parser.parseOptionalKeyword("cleanup"))) - result.addAttribute("cleanup", parser.getBuilder().getUnitAttr()); - - // Parse clauses with types - while (succeeded(parser.parseOptionalLParen()) && - (succeeded(parser.parseOptionalKeyword("filter")) || - succeeded(parser.parseOptionalKeyword("catch")))) { - OpAsmParser::UnresolvedOperand operand; - Type ty; - if (parser.parseOperand(operand) || parser.parseColon() || - parser.parseType(ty) || - parser.resolveOperand(operand, ty, result.operands) || - parser.parseRParen()) - return failure(); - } - - Type type; - if (parser.parseColon() || parser.parseType(type)) - return failure(); - - result.addTypes(type); - return success(); -} - -//===----------------------------------------------------------------------===// -// CallOp -//===----------------------------------------------------------------------===// - -void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, - StringRef callee, ValueRange args) { - build(builder, state, results, builder.getStringAttr(callee), args); -} - -void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, - StringAttr callee, ValueRange args) { - build(builder, state, results, SymbolRefAttr::get(callee), args, nullptr, - nullptr); -} - -void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results, - FlatSymbolRefAttr callee, ValueRange args) { - build(builder, state, results, callee, args, nullptr, nullptr); -} - -void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func, - ValueRange args) { - SmallVector results; - Type resultType = func.getFunctionType().getReturnType(); - if (!resultType.isa()) - results.push_back(resultType); - build(builder, state, results, SymbolRefAttr::get(func), args, nullptr, - nullptr); -} - -CallInterfaceCallable CallOp::getCallableForCallee() { - // Direct call. - if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) - return calleeAttr; - // Indirect call, callee Value is the first operand. - return getOperand(0); -} - -Operation::operand_range CallOp::getArgOperands() { - return getOperands().drop_front(getCallee().has_value() ? 0 : 1); -} - -LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { - if (getNumResults() > 1) - return emitOpError("must have 0 or 1 result"); - - // Type for the callee, we'll get it differently depending if it is a direct - // or indirect call. - Type fnType; - - bool isIndirect = false; - - // If this is an indirect call, the callee attribute is missing. - FlatSymbolRefAttr calleeName = getCalleeAttr(); - if (!calleeName) { - isIndirect = true; - if (!getNumOperands()) - return emitOpError( - "must have either a `callee` attribute or at least an operand"); - auto ptrType = getOperand(0).getType().dyn_cast(); - if (!ptrType) - return emitOpError("indirect call expects a pointer as callee: ") - << ptrType; - - if (ptrType.isOpaque()) - return success(); - - fnType = ptrType.getElementType(); - } else { - Operation *callee = - symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr()); - if (!callee) - return emitOpError() - << "'" << calleeName.getValue() - << "' does not reference a symbol in the current scope"; - auto fn = dyn_cast(callee); - if (!fn) - return emitOpError() << "'" << calleeName.getValue() - << "' does not reference a valid LLVM function"; - - fnType = fn.getFunctionType(); - } - - LLVMFunctionType funcType = fnType.dyn_cast(); - if (!funcType) - return emitOpError("callee does not have a functional type: ") << fnType; - - // Verify that the operand and result types match the callee. - - if (!funcType.isVarArg() && - funcType.getNumParams() != (getNumOperands() - isIndirect)) - return emitOpError() << "incorrect number of operands (" - << (getNumOperands() - isIndirect) - << ") for callee (expecting: " - << funcType.getNumParams() << ")"; - - if (funcType.getNumParams() > (getNumOperands() - isIndirect)) - return emitOpError() << "incorrect number of operands (" - << (getNumOperands() - isIndirect) - << ") for varargs callee (expecting at least: " - << funcType.getNumParams() << ")"; - - for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i) - if (getOperand(i + isIndirect).getType() != funcType.getParamType(i)) - return emitOpError() << "operand type mismatch for operand " << i << ": " - << getOperand(i + isIndirect).getType() - << " != " << funcType.getParamType(i); - - if (getNumResults() == 0 && - !funcType.getReturnType().isa()) - return emitOpError() << "expected function call to produce a value"; - - if (getNumResults() != 0 && - funcType.getReturnType().isa()) - return emitOpError() - << "calling function with void result must not produce values"; - - if (getNumResults() > 1) - return emitOpError() - << "expected LLVM function call to produce 0 or 1 result"; - - if (getNumResults() && getResult().getType() != funcType.getReturnType()) - return emitOpError() << "result type mismatch: " << getResult().getType() - << " != " << funcType.getReturnType(); - + // NullOp and AddressOfOp allowed + if (value.getDefiningOp()) + continue; + if (value.getDefiningOp()) + continue; + return emitError("clause #") + << idx << " is not a known constant - null, addressof, bitcast"; + } + } return success(); } -void CallOp::print(OpAsmPrinter &p) { - auto callee = getCallee(); - bool isDirect = callee.has_value(); +void LandingpadOp::print(OpAsmPrinter &p) { + p << (getCleanup() ? " cleanup " : " "); - // Print the direct callee if present as a function attribute, or an indirect - // callee (first operand) otherwise. - p << ' '; - if (isDirect) - p.printSymbolName(callee.value()); - else - p << getOperand(0); + // Clauses + for (auto value : getOperands()) { + // Similar to llvm - if clause is an array type then it is filter + // clause else catch clause + bool isArrayTy = value.getType().isa(); + p << '(' << (isArrayTy ? "filter " : "catch ") << value << " : " + << value.getType() << ") "; + } - auto args = getOperands().drop_front(isDirect ? 0 : 1); - p << '(' << args << ')'; - p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"callee"}); + p.printOptionalAttrDict((*this)->getAttrs(), {"cleanup"}); - // Reconstruct the function MLIR function type from operand and result types. - p << " : "; - p.printFunctionalType(args.getTypes(), getResultTypes()); + p << ": " << getType(); } -// ::= `llvm.call` (function-id | ssa-use) `(` ssa-use-list `)` -// attribute-dict? `:` function-type -ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) { - SmallVector operands; - Type type; - SymbolRefAttr funcAttr; - SMLoc trailingTypeLoc; - - // Parse an operand list that will, in practice, contain 0 or 1 operand. In - // case of an indirect call, there will be 1 operand before `(`. In case of a - // direct call, there will be no operands and the parser will stop at the - // function identifier without complaining. - if (parser.parseOperandList(operands)) - return failure(); - bool isDirect = operands.empty(); +// ::= `llvm.landingpad` `cleanup`? +// ((`catch` | `filter`) operand-type ssa-use)* attribute-dict? +ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) { + // Check for cleanup + if (succeeded(parser.parseOptionalKeyword("cleanup"))) + result.addAttribute("cleanup", parser.getBuilder().getUnitAttr()); - // Optionally parse a function identifier. - if (isDirect) - if (parser.parseAttribute(funcAttr, "callee", result.attributes)) + // Parse clauses with types + while (succeeded(parser.parseOptionalLParen()) && + (succeeded(parser.parseOptionalKeyword("filter")) || + succeeded(parser.parseOptionalKeyword("catch")))) { + OpAsmParser::UnresolvedOperand operand; + Type ty; + if (parser.parseOperand(operand) || parser.parseColon() || + parser.parseType(ty) || + parser.resolveOperand(operand, ty, result.operands) || + parser.parseRParen()) return failure(); + } - if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) || - parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() || - parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type)) + Type type; + if (parser.parseColon() || parser.parseType(type)) return failure(); - auto funcType = type.dyn_cast(); - if (!funcType) - return parser.emitError(trailingTypeLoc, "expected function type"); - if (funcType.getNumResults() > 1) - return parser.emitError(trailingTypeLoc, - "expected function with 0 or 1 result"); - if (isDirect) { - // Make sure types match. - if (parser.resolveOperands(operands, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); - if (funcType.getNumResults() != 0 && - !funcType.getResult(0).isa()) - result.addTypes(funcType.getResults()); - } else { - Builder &builder = parser.getBuilder(); - Type llvmResultType; - if (funcType.getNumResults() == 0) { - llvmResultType = LLVM::LLVMVoidType::get(builder.getContext()); - } else { - llvmResultType = funcType.getResult(0); - if (!isCompatibleType(llvmResultType)) - return parser.emitError(trailingTypeLoc, - "expected result to have LLVM type"); - } - - SmallVector argTypes; - argTypes.reserve(funcType.getNumInputs()); - for (int i = 0, e = funcType.getNumInputs(); i < e; ++i) { - auto argType = funcType.getInput(i); - if (!isCompatibleType(argType)) - return parser.emitError(trailingTypeLoc, - "expected LLVM types as inputs"); - argTypes.push_back(argType); - } - auto llvmFuncType = LLVM::LLVMFunctionType::get(llvmResultType, argTypes); - auto wrappedFuncType = LLVM::LLVMPointerType::get(llvmFuncType); - - auto funcArguments = - ArrayRef(operands).drop_front(); - - // Make sure that the first operand (indirect callee) matches the wrapped - // LLVM IR function type, and that the types of the other call operands - // match the types of the function arguments. - if (parser.resolveOperand(operands[0], wrappedFuncType, result.operands) || - parser.resolveOperands(funcArguments, funcType.getInputs(), - parser.getNameLoc(), result.operands)) - return failure(); - - if (!llvmResultType.isa()) - result.addTypes(llvmResultType); - } - + result.addTypes(type); return success(); } @@ -2259,90 +2317,16 @@ LogicalResult LLVM::ConstantOp::verify() { OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); } //===----------------------------------------------------------------------===// -// Utility functions for parsing atomic ops -//===----------------------------------------------------------------------===// - -// Helper function to parse a keyword into the specified attribute named by -// `attrName`. The keyword must match one of the string values defined by the -// AtomicBinOp enum. The resulting I64 attribute is added to the `result` -// state. -static ParseResult parseAtomicBinOp(OpAsmParser &parser, OperationState &result, - StringRef attrName) { - SMLoc loc; - StringRef keyword; - if (parser.getCurrentLocation(&loc) || parser.parseKeyword(&keyword)) - return failure(); - - // Replace the keyword `keyword` with an integer attribute. - auto kind = symbolizeAtomicBinOp(keyword); - if (!kind) { - return parser.emitError(loc) - << "'" << keyword << "' is an incorrect value of the '" << attrName - << "' attribute"; - } - - auto value = static_cast(*kind); - auto attr = parser.getBuilder().getI64IntegerAttr(value); - result.addAttribute(attrName, attr); - - return success(); -} - -// Helper function to parse a keyword into the specified attribute named by -// `attrName`. The keyword must match one of the string values defined by the -// AtomicOrdering enum. The resulting I64 attribute is added to the `result` -// state. -static ParseResult parseAtomicOrdering(OpAsmParser &parser, - OperationState &result, - StringRef attrName) { - SMLoc loc; - StringRef ordering; - if (parser.getCurrentLocation(&loc) || parser.parseKeyword(&ordering)) - return failure(); - - // Replace the keyword `ordering` with an integer attribute. - auto kind = symbolizeAtomicOrdering(ordering); - if (!kind) { - return parser.emitError(loc) - << "'" << ordering << "' is an incorrect value of the '" << attrName - << "' attribute"; - } - - auto value = static_cast(*kind); - auto attr = parser.getBuilder().getI64IntegerAttr(value); - result.addAttribute(attrName, attr); - - return success(); -} - -//===----------------------------------------------------------------------===// -// Printer, parser and verifier for LLVM::AtomicRMWOp. +// AtomicRMWOp //===----------------------------------------------------------------------===// -void AtomicRMWOp::print(OpAsmPrinter &p) { - p << ' ' << stringifyAtomicBinOp(getBinOp()) << ' ' << getPtr() << ", " - << getVal() << ' ' << stringifyAtomicOrdering(getOrdering()) << ' '; - p.printOptionalAttrDict((*this)->getAttrs(), {"bin_op", "ordering"}); - p << " : " << getRes().getType(); -} - -// ::= `llvm.atomicrmw` keyword ssa-use `,` ssa-use keyword -// attribute-dict? `:` type -ParseResult AtomicRMWOp::parse(OpAsmParser &parser, OperationState &result) { - Type type; - OpAsmParser::UnresolvedOperand ptr, val; - if (parseAtomicBinOp(parser, result, "bin_op") || parser.parseOperand(ptr) || - parser.parseComma() || parser.parseOperand(val) || - parseAtomicOrdering(parser, result, "ordering") || - parser.parseOptionalAttrDict(result.attributes) || - parser.parseColonType(type) || - parser.resolveOperand(ptr, LLVM::LLVMPointerType::get(type), - result.operands) || - parser.resolveOperand(val, type, result.operands)) - return failure(); - - result.addTypes(type); - return success(); +void AtomicRMWOp::build(OpBuilder &builder, OperationState &state, + AtomicBinOp binOp, Value ptr, Value val, + AtomicOrdering ordering, StringRef syncscope, + unsigned alignment, bool isVolatile) { + build(builder, state, val.getType(), binOp, ptr, val, ordering, + !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile); } LogicalResult AtomicRMWOp::verify() { @@ -2351,20 +2335,12 @@ LogicalResult AtomicRMWOp::verify() { if (!ptrType.isOpaque() && valType != ptrType.getElementType()) return emitOpError("expected LLVM IR element type for operand #0 to " "match type for operand #1"); - auto resType = getRes().getType(); - if (resType != valType) - return emitOpError( - "expected LLVM IR result type to match type for operand #1"); - if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub) { + if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub || + getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax) { if (!mlir::LLVM::isCompatibleFloatingPointType(valType)) return emitOpError("expected LLVM IR floating point type"); } else if (getBinOp() == AtomicBinOp::xchg) { - auto intType = valType.dyn_cast(); - unsigned intBitWidth = intType ? intType.getWidth() : 0; - if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 && - intBitWidth != 64 && !valType.isa() && - !valType.isa() && !valType.isa() && - !valType.isa()) + if (!isTypeCompatibleWithAtomicOp(valType, /*isPointerTypeAllowed=*/false)) return emitOpError("unexpected LLVM IR type for 'xchg' bin_op"); } else { auto intType = valType.dyn_cast(); @@ -2384,63 +2360,37 @@ LogicalResult AtomicRMWOp::verify() { } //===----------------------------------------------------------------------===// -// Printer, parser and verifier for LLVM::AtomicCmpXchgOp. +// AtomicCmpXchgOp //===----------------------------------------------------------------------===// -void AtomicCmpXchgOp::print(OpAsmPrinter &p) { - p << ' ' << getPtr() << ", " << getCmp() << ", " << getVal() << ' ' - << stringifyAtomicOrdering(getSuccessOrdering()) << ' ' - << stringifyAtomicOrdering(getFailureOrdering()); - p.printOptionalAttrDict((*this)->getAttrs(), - {"success_ordering", "failure_ordering"}); - p << " : " << getVal().getType(); +/// Returns an LLVM struct type that contains a value type and a boolean type. +static LLVMStructType getValAndBoolStructType(Type valType) { + auto boolType = IntegerType::get(valType.getContext(), 1); + return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType}); } -// ::= `llvm.cmpxchg` ssa-use `,` ssa-use `,` ssa-use -// keyword keyword attribute-dict? `:` type -ParseResult AtomicCmpXchgOp::parse(OpAsmParser &parser, - OperationState &result) { - auto &builder = parser.getBuilder(); - Type type; - OpAsmParser::UnresolvedOperand ptr, cmp, val; - if (parser.parseOperand(ptr) || parser.parseComma() || - parser.parseOperand(cmp) || parser.parseComma() || - parser.parseOperand(val) || - parseAtomicOrdering(parser, result, "success_ordering") || - parseAtomicOrdering(parser, result, "failure_ordering") || - parser.parseOptionalAttrDict(result.attributes) || - parser.parseColonType(type) || - parser.resolveOperand(ptr, LLVM::LLVMPointerType::get(type), - result.operands) || - parser.resolveOperand(cmp, type, result.operands) || - parser.resolveOperand(val, type, result.operands)) - return failure(); - - auto boolType = IntegerType::get(builder.getContext(), 1); - auto resultType = - LLVMStructType::getLiteral(builder.getContext(), {type, boolType}); - result.addTypes(resultType); - - return success(); +void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state, + Value ptr, Value cmp, Value val, + AtomicOrdering successOrdering, + AtomicOrdering failureOrdering, StringRef syncscope, + unsigned alignment, bool isWeak, bool isVolatile) { + build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val, + successOrdering, failureOrdering, + !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak, + isVolatile); } LogicalResult AtomicCmpXchgOp::verify() { auto ptrType = getPtr().getType().cast(); if (!ptrType) return emitOpError("expected LLVM IR pointer type for operand #0"); - auto cmpType = getCmp().getType(); auto valType = getVal().getType(); - if (cmpType != valType) - return emitOpError("expected both value operands to have the same type"); - if (!ptrType.isOpaque() && cmpType != ptrType.getElementType()) + if (!ptrType.isOpaque() && valType != ptrType.getElementType()) return emitOpError("expected LLVM IR element type for operand #0 to " "match type for all other operands"); - auto intType = valType.dyn_cast(); - unsigned intBitWidth = intType ? intType.getWidth() : 0; - if (!valType.isa() && intBitWidth != 8 && - intBitWidth != 16 && intBitWidth != 32 && intBitWidth != 64 && - !valType.isa() && !valType.isa() && - !valType.isa() && !valType.isa()) + if (!isTypeCompatibleWithAtomicOp(valType, + /*isPointerTypeAllowed=*/true)) return emitOpError("unexpected LLVM IR type"); if (getSuccessOrdering() < AtomicOrdering::monotonic || getFailureOrdering() < AtomicOrdering::monotonic) @@ -2452,35 +2402,13 @@ LogicalResult AtomicCmpXchgOp::verify() { } //===----------------------------------------------------------------------===// -// Printer, parser and verifier for LLVM::FenceOp. +// FenceOp //===----------------------------------------------------------------------===// -// ::= `llvm.fence` (`syncscope(`strAttr`)`)? keyword -// attribute-dict? -ParseResult FenceOp::parse(OpAsmParser &parser, OperationState &result) { - StringAttr sScope; - StringRef syncscopeKeyword = "syncscope"; - if (!failed(parser.parseOptionalKeyword(syncscopeKeyword))) { - if (parser.parseLParen() || - parser.parseAttribute(sScope, syncscopeKeyword, result.attributes) || - parser.parseRParen()) - return failure(); - } else { - result.addAttribute(syncscopeKeyword, - parser.getBuilder().getStringAttr("")); - } - if (parseAtomicOrdering(parser, result, "ordering") || - parser.parseOptionalAttrDict(result.attributes)) - return failure(); - return success(); -} - -void FenceOp::print(OpAsmPrinter &p) { - StringRef syncscopeKeyword = "syncscope"; - p << ' '; - if (!(*this)->getAttr(syncscopeKeyword).cast().getValue().empty()) - p << "syncscope(" << (*this)->getAttr(syncscopeKeyword) << ") "; - p << stringifyAtomicOrdering(getOrdering()); +void FenceOp::build(OpBuilder &builder, OperationState &state, + AtomicOrdering ordering, StringRef syncscope) { + build(builder, state, ordering, + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); } LogicalResult FenceOp::verify() { @@ -2493,7 +2421,7 @@ LogicalResult FenceOp::verify() { } //===----------------------------------------------------------------------===// -// Folder for LLVM::BitcastOp +// Folder and verifier for LLVM::BitcastOp //===----------------------------------------------------------------------===// OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) { @@ -2507,6 +2435,41 @@ OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) { return {}; } +LogicalResult LLVM::BitcastOp::verify() { + auto resultType = extractVectorElementType(getResult().getType()) + .dyn_cast(); + auto sourceType = + extractVectorElementType(getArg().getType()).dyn_cast(); + + // If one of the types is a pointer (or vector of pointers), then + // both source and result type have to be pointers. + if (static_cast(resultType) != static_cast(sourceType)) + return emitOpError("can only cast pointers from and to pointers"); + + if (!resultType) + return success(); + + auto isVector = [](Type type) { + return type.isa(); + }; + + // Due to bitcast requiring both operands to be of the same size, it is not + // possible for only one of the two to be a pointer of vectors. + if (isVector(getResult().getType()) && !isVector(getArg().getType())) + return emitOpError("cannot cast pointer to vector of pointers"); + + if (!isVector(getResult().getType()) && isVector(getArg().getType())) + return emitOpError("cannot cast vector of pointers to pointer"); + + // Bitcast cannot cast between pointers of different address spaces. + // 'llvm.addrspacecast' must be used for this purpose instead. + if (resultType.getAddressSpace() != sourceType.getAddressSpace()) + return emitOpError("cannot cast pointers of different address spaces, " + "use 'llvm.addrspacecast' instead"); + + return success(); +} + //===----------------------------------------------------------------------===// // Folder for LLVM::AddrSpaceCastOp //===----------------------------------------------------------------------===// @@ -2626,29 +2589,28 @@ struct TBAAGraphNode { // TBAA graph. class TBAAGraph { - // Mapping between symbol names defined by TBAA - // operations and corresponding TBAAGraphNode's. - DenseMap nodeMap; - // Synthetic root node that has all graph nodes - // in its operands list. - TBAAGraphNode root; - public: using iterator = SmallVectorImpl::iterator; + // Creates a new graph with nodes corresponding to `symbolNames` defined by a + // set of TBAA operations. + TBAAGraph(ArrayRef symbolNames) { + for (auto symbol : symbolNames) { + TBAAGraphNode &node = nodeMap[symbol]; + assert(node.symbol.empty() && "node is already in the graph"); + node.symbol = symbol; + } + + // Fill the graph operands once all nodes were added. Otherwise, + // reallocation can lead to pointer invalidation. + for (auto symbol : symbolNames) + root.operands.push_back(&nodeMap[symbol]); + } + iterator begin() { return root.operands.begin(); } iterator end() { return root.operands.end(); } TBAAGraphNode *getEntryNode() { return &root; } - // Add new graph node corresponding to `symbol` - // defined by a TBAA operation. - void addNodeDefinition(StringAttr symbol) { - TBAAGraphNode &node = nodeMap[symbol]; - assert(node.symbol.empty() && "node is already in the graph"); - node.symbol = symbol; - root.operands.push_back(&node); - } - // Get a pointer to TBAAGraphNode corresponding // to `symbol`. The node must be already in the graph. TBAAGraphNode *operator[](StringAttr symbol) { @@ -2656,8 +2618,17 @@ class TBAAGraph { assert(it != nodeMap.end() && "node must be in the graph"); return &it->second; } + +private: + // Mapping between symbol names defined by TBAA + // operations and corresponding TBAAGraphNode's. + DenseMap nodeMap; + // Synthetic root node that has all graph nodes + // in its operands list. + TBAAGraphNode root; }; } // end anonymous namespace + namespace llvm { // GraphTraits definitions for using TBAAGraph with // scc_iterator. @@ -2689,20 +2660,26 @@ LogicalResult MetadataOp::verifyRegions() { Region &body = getBody(); // Symbol names defined by TBAARootMetadataOp and TBAATypeDescriptorOp. llvm::SmallDenseSet definedGraphSymbols; - // Complete TBAA graph consisting of TBAARootMetadataOp, - // TBAATypeDescriptorOp, and TBAATagOp symbols. It is used - // for detecting cycles in the TBAA graph, which is illegal. - TBAAGraph tbaaGraph; - for (Operation &op : body.getOps()) + // Collection of symbol names to ensure a stable ordering of the pointers. + // Otherwise, error messages might not be deterministic. + SmallVector symbolNames; + + for (Operation &op : body.getOps()) { if (isa(op) || isa(op)) { StringAttr symbolDef = cast(op).getNameAttr(); definedGraphSymbols.insert(symbolDef); - tbaaGraph.addNodeDefinition(symbolDef); + symbolNames.push_back(symbolDef); } else if (auto tagOp = dyn_cast(op)) { - tbaaGraph.addNodeDefinition(tagOp.getSymNameAttr()); + symbolNames.push_back(tagOp.getSymNameAttr()); } + } + + // Complete TBAA graph consisting of TBAARootMetadataOp, + // TBAATypeDescriptorOp, and TBAATagOp symbols. It is used + // for detecting cycles in the TBAA graph, which is illegal. + TBAAGraph tbaaGraph(symbolNames); // Verify that TBAA metadata operations refer symbols // from definedGraphSymbols only. Note that TBAATagOp @@ -2822,6 +2799,21 @@ LogicalResult TBAATypeDescriptorOp::verify() { return success(); } +//===----------------------------------------------------------------------===// +// AliasScopeMetadataOp +//===----------------------------------------------------------------------===// + +LogicalResult AliasScopeMetadataOp::verify() { + Operation *domainOp = SymbolTable::lookupNearestSymbolFrom( + this->getOperation(), getDomainAttr()); + if (!isa_and_nonnull(domainOp)) { + return this->emitOpError() + << "expected '" << getDomain() + << "' to reference a domain operation in the same region"; + } + return success(); +} + //===----------------------------------------------------------------------===// // OpAsmDialectInterface //===----------------------------------------------------------------------===// @@ -2832,10 +2824,14 @@ struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface { AliasResult getAlias(Attribute attr, raw_ostream &os) const override { return TypeSwitch(attr) - .Case([&](auto attr) { + .Case([&](auto attr) { os << decltype(attr)::getMnemonic(); return AliasResult::OverridableAlias; }) @@ -3024,56 +3020,6 @@ LogicalResult LLVMDialect::verifyDataLayoutString( /// Verify LLVM dialect attributes. LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op, NamedAttribute attr) { - // If the `llvm.loop` attribute is present, enforce the following structure, - // which the module translation can assume. - if (attr.getName() == LLVMDialect::getLoopAttrName()) { - auto loopAttr = attr.getValue().dyn_cast(); - if (!loopAttr) - return op->emitOpError() << "expected '" << LLVMDialect::getLoopAttrName() - << "' to be a dictionary attribute"; - std::optional parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - auto accessGroups = parallelAccessGroup->getValue().dyn_cast(); - if (!accessGroups) - return op->emitOpError() - << "expected '" << LLVMDialect::getParallelAccessAttrName() - << "' to be an array attribute"; - for (Attribute attr : accessGroups) { - auto accessGroupRef = attr.dyn_cast(); - if (!accessGroupRef) - return op->emitOpError() - << "expected '" << attr << "' to be a symbol reference"; - StringAttr metadataName = accessGroupRef.getRootReference(); - auto metadataOp = - SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << attr << "' to reference a metadata op"; - StringAttr accessGroupName = accessGroupRef.getLeafReference(); - Operation *accessGroupOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); - if (!accessGroupOp) - return op->emitOpError() - << "expected '" << attr << "' to reference an access_group op"; - } - } - - std::optional loopOptions = - loopAttr.getNamed(LLVMDialect::getLoopOptionsAttrName()); - if (loopOptions && !loopOptions->getValue().isa()) - return op->emitOpError() - << "expected '" << LLVMDialect::getLoopOptionsAttrName() - << "' to be a `loopopts` attribute"; - } - - if (attr.getName() == LLVMDialect::getStructAttrsAttrName()) { - return op->emitOpError() - << "'" << LLVM::LLVMDialect::getStructAttrsAttrName() - << "' is permitted only in argument or result attributes"; - } - // If the data layout attribute is present, it must use the LLVM data layout // syntax. Try parsing it and report errors in case of failure. Users of this // attribute may assume it is well-formed and can pass it to the (asserting) @@ -3090,44 +3036,115 @@ LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op, << "' to be a string attributes"; } -LogicalResult LLVMDialect::verifyStructAttr(Operation *op, Attribute attr, - Type annotatedType) { - auto structType = annotatedType.dyn_cast(); - if (!structType) { - const auto emitIncorrectAnnotatedType = [&op]() { +LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op, + Type paramType, + NamedAttribute paramAttr) { + // LLVM attribute may be attached to a result of operation that has not been + // converted to LLVM dialect yet, so the result may have a type with unknown + // representation in LLVM dialect type space. In this case we cannot verify + // whether the attribute may be + bool verifyValueType = isCompatibleType(paramType); + StringAttr name = paramAttr.getName(); + + auto checkUnitAttrType = [&]() -> LogicalResult { + if (!paramAttr.getValue().isa()) + return op->emitError() << name << " should be a unit attribute"; + return success(); + }; + auto checkTypeAttrType = [&]() -> LogicalResult { + if (!paramAttr.getValue().isa()) + return op->emitError() << name << " should be a type attribute"; + return success(); + }; + auto checkIntegerAttrType = [&]() -> LogicalResult { + if (!paramAttr.getValue().isa()) + return op->emitError() << name << " should be an integer attribute"; + return success(); + }; + auto checkPointerType = [&]() -> LogicalResult { + if (!paramType.isa()) return op->emitError() - << "expected '" << LLVMDialect::getStructAttrsAttrName() - << "' to annotate '!llvm.struct' or '!llvm.ptr>'"; - }; - const auto ptrType = annotatedType.dyn_cast(); - if (!ptrType) - return emitIncorrectAnnotatedType(); - structType = ptrType.getElementType().dyn_cast(); - if (!structType) - return emitIncorrectAnnotatedType(); - } - - const auto arrAttrs = attr.dyn_cast(); - if (!arrAttrs) - return op->emitError() << "expected '" - << LLVMDialect::getStructAttrsAttrName() - << "' to be an array attribute"; - - if (structType.getBody().size() != arrAttrs.size()) - return op->emitError() - << "size of '" << LLVMDialect::getStructAttrsAttrName() - << "' must match the size of the annotated '!llvm.struct'"; - return success(); -} + << name << " attribute attached to non-pointer LLVM type"; + return success(); + }; + auto checkIntegerType = [&]() -> LogicalResult { + if (!paramType.isa()) + return op->emitError() + << name << " attribute attached to non-integer LLVM type"; + return success(); + }; + auto checkPointerTypeMatches = [&]() -> LogicalResult { + if (failed(checkPointerType())) + return failure(); + auto ptrType = paramType.cast(); + auto typeAttr = paramAttr.getValue().cast(); + + if (!ptrType.isOpaque() && ptrType.getElementType() != typeAttr.getValue()) + return op->emitError() + << name + << " attribute attached to LLVM pointer argument of " + "different type"; + return success(); + }; + + // Check a unit attribute that is attached to a pointer value. + if (name == LLVMDialect::getNoAliasAttrName() || + name == LLVMDialect::getReadonlyAttrName() || + name == LLVMDialect::getReadnoneAttrName() || + name == LLVMDialect::getWriteOnlyAttrName() || + name == LLVMDialect::getNestAttrName() || + name == LLVMDialect::getNoCaptureAttrName() || + name == LLVMDialect::getNoFreeAttrName() || + name == LLVMDialect::getNonNullAttrName()) { + if (failed(checkUnitAttrType())) + return failure(); + if (verifyValueType && failed(checkPointerType())) + return failure(); + return success(); + } + + // Check a type attribute that is attached to a pointer value. + if (name == LLVMDialect::getStructRetAttrName() || + name == LLVMDialect::getByValAttrName() || + name == LLVMDialect::getByRefAttrName() || + name == LLVMDialect::getInAllocaAttrName() || + name == LLVMDialect::getPreallocatedAttrName()) { + if (failed(checkTypeAttrType())) + return failure(); + if (verifyValueType && failed(checkPointerTypeMatches())) + return failure(); + return success(); + } -static LogicalResult verifyFuncOpInterfaceStructAttr( - Operation *op, Attribute attr, - const std::function &getAnnotatedType) { - if (auto funcOp = dyn_cast(op)) - return LLVMDialect::verifyStructAttr(op, attr, getAnnotatedType(funcOp)); - return op->emitError() << "expected '" - << LLVMDialect::getStructAttrsAttrName() - << "' to be used on function-like operations"; + // Check a unit attribute that is attached to an integer value. + if (name == LLVMDialect::getSExtAttrName() || + name == LLVMDialect::getZExtAttrName()) { + if (failed(checkUnitAttrType())) + return failure(); + if (verifyValueType && failed(checkIntegerType())) + return failure(); + return success(); + } + + // Check an integer attribute that is attached to a pointer value. + if (name == LLVMDialect::getAlignAttrName() || + name == LLVMDialect::getDereferenceableAttrName() || + name == LLVMDialect::getDereferenceableOrNullAttrName() || + name == LLVMDialect::getStackAlignmentAttrName()) { + if (failed(checkIntegerAttrType())) + return failure(); + if (verifyValueType && failed(checkPointerType())) + return failure(); + return success(); + } + + // Check a unit attribute that can be attached to arbitrary types. + if (name == LLVMDialect::getNoUndefAttrName() || + name == LLVMDialect::getInRegAttrName() || + name == LLVMDialect::getReturnedAttrName()) + return checkUnitAttrType(); + + return success(); } /// Verify LLVMIR function argument attributes. @@ -3135,108 +3152,49 @@ LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op, unsigned regionIdx, unsigned argIdx, NamedAttribute argAttr) { - // Check that llvm.noalias is a unit attribute. - if (argAttr.getName() == LLVMDialect::getNoAliasAttrName() && - !argAttr.getValue().isa()) - return op->emitError() - << "expected llvm.noalias argument attribute to be a unit attribute"; - // Check that llvm.align is an integer attribute. - if (argAttr.getName() == LLVMDialect::getAlignAttrName() && - !argAttr.getValue().isa()) - return op->emitError() - << "llvm.align argument attribute of non integer type"; - if (argAttr.getName() == LLVMDialect::getStructAttrsAttrName()) { - return verifyFuncOpInterfaceStructAttr( - op, argAttr.getValue(), [argIdx](FunctionOpInterface funcOp) { - return funcOp.getArgumentTypes()[argIdx]; - }); - } - return success(); + auto funcOp = dyn_cast(op); + if (!funcOp) + return success(); + Type argType = funcOp.getArgumentTypes()[argIdx]; + + return verifyParameterAttribute(op, argType, argAttr); } LogicalResult LLVMDialect::verifyRegionResultAttribute(Operation *op, unsigned regionIdx, unsigned resIdx, NamedAttribute resAttr) { - StringAttr name = resAttr.getName(); - if (name == LLVMDialect::getStructAttrsAttrName()) { - return verifyFuncOpInterfaceStructAttr( - op, resAttr.getValue(), [resIdx](FunctionOpInterface funcOp) { - return funcOp.getResultTypes()[resIdx]; - }); - } - if (auto funcOp = dyn_cast(op)) { - mlir::Type resTy = funcOp.getResultTypes()[resIdx]; - - // Check to see if this function has a void return with a result attribute - // to it. It isn't clear what semantics we would assign to that. - if (resTy.isa()) - return op->emitError() << "cannot attach result attributes to functions " - "with a void return"; - - // LLVM attribute may be attached to a result of operation - // that has not been converted to LLVM dialect yet, so the result - // may have a type with unknown representation in LLVM dialect type - // space. In this case we cannot verify whether the attribute may be - // attached to a result of such type. - bool verifyValueType = isCompatibleType(resTy); - Attribute attrValue = resAttr.getValue(); - - // TODO: get rid of code duplication here and in verifyRegionArgAttribute(). - if (name == LLVMDialect::getAlignAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.align result attribute to be " - "an integer attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.align attribute attached to non-pointer result"; - return success(); - } - if (name == LLVMDialect::getNoAliasAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.noalias result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.noalias attribute attached to non-pointer result"; - return success(); - } - if (name == LLVMDialect::getReadonlyAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.readonly result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.readonly attribute attached to non-pointer result"; - return success(); - } - if (name == LLVMDialect::getNoUndefAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.noundef result attribute to " - "be a unit attribute"; - return success(); - } - if (name == LLVMDialect::getSExtAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.signext result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.signext attribute attached to non-integer result"; - return success(); - } - if (name == LLVMDialect::getZExtAttrName()) { - if (!attrValue.isa()) - return op->emitError() << "expected llvm.zeroext result attribute to " - "be a unit attribute"; - if (verifyValueType && !resTy.isa()) - return op->emitError() - << "llvm.zeroext attribute attached to non-integer result"; - return success(); - } - } - - return success(); + auto funcOp = dyn_cast(op); + if (!funcOp) + return success(); + Type resType = funcOp.getResultTypes()[resIdx]; + + // Check to see if this function has a void return with a result attribute + // to it. It isn't clear what semantics we would assign to that. + if (resType.isa()) + return op->emitError() << "cannot attach result attributes to functions " + "with a void return"; + + // Check to see if this attribute is allowed as a result attribute. Only + // explicitly forbidden LLVM attributes will cause an error. + auto name = resAttr.getName(); + if (name == LLVMDialect::getAllocAlignAttrName() || + name == LLVMDialect::getAllocatedPointerAttrName() || + name == LLVMDialect::getByValAttrName() || + name == LLVMDialect::getByRefAttrName() || + name == LLVMDialect::getInAllocaAttrName() || + name == LLVMDialect::getNestAttrName() || + name == LLVMDialect::getNoCaptureAttrName() || + name == LLVMDialect::getNoFreeAttrName() || + name == LLVMDialect::getPreallocatedAttrName() || + name == LLVMDialect::getReadnoneAttrName() || + name == LLVMDialect::getReadonlyAttrName() || + name == LLVMDialect::getReturnedAttrName() || + name == LLVMDialect::getStackAlignmentAttrName() || + name == LLVMDialect::getStructRetAttrName() || + name == LLVMDialect::getWriteOnlyAttrName()) + return op->emitError() << name << " is not a valid result attribute"; + return verifyParameterAttribute(op, resType, resAttr); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp b/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp index b0e884c285f0..81b9b279a650 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp +++ b/mlir/lib/Dialect/MemRef/Transforms/EmulateWideInt.cpp @@ -66,7 +66,8 @@ struct ConvertMemRefLoad final : OpConversionPattern { op.getMemRefType())); rewriter.replaceOpWithNewOp( - op, newResTy, adaptor.getMemref(), adaptor.getIndices()); + op, newResTy, adaptor.getMemref(), adaptor.getIndices(), + op.getNontemporal()); return success(); } }; @@ -88,7 +89,8 @@ struct ConvertMemRefStore final : OpConversionPattern { op.getMemRefType())); rewriter.replaceOpWithNewOp( - op, adaptor.getValue(), adaptor.getMemref(), adaptor.getIndices()); + op, adaptor.getValue(), adaptor.getMemref(), adaptor.getIndices(), + op.getNontemporal()); return success(); } }; diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp index 92f02c068d2b..ff297a9cd98f 100644 --- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp +++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp @@ -384,10 +384,14 @@ LogicalResult LoadOpOfSubViewOpFolder::matchAndRewrite( return failure(); llvm::TypeSwitch(loadOp) - .Case([&](auto op) { - rewriter.replaceOpWithNewOp(loadOp, subViewOp.getSource(), + .Case([&](AffineLoadOp op) { + rewriter.replaceOpWithNewOp(loadOp, subViewOp.getSource(), sourceIndices); }) + .Case([&](memref::LoadOp op) { + rewriter.replaceOpWithNewOp( + loadOp, subViewOp.getSource(), sourceIndices, op.getNontemporal()); + }) .Case([&](vector::TransferReadOp transferReadOp) { rewriter.replaceOpWithNewOp( transferReadOp, transferReadOp.getVectorType(), @@ -490,10 +494,15 @@ LogicalResult StoreOpOfSubViewOpFolder::matchAndRewrite( return failure(); llvm::TypeSwitch(storeOp) - .Case([&](auto op) { - rewriter.replaceOpWithNewOp( + .Case([&](AffineStoreOp op) { + rewriter.replaceOpWithNewOp( storeOp, storeOp.getValue(), subViewOp.getSource(), sourceIndices); }) + .Case([&](memref::StoreOp op) { + rewriter.replaceOpWithNewOp( + storeOp, storeOp.getValue(), subViewOp.getSource(), sourceIndices, + op.getNontemporal()); + }) .Case([&](vector::TransferWriteOp op) { rewriter.replaceOpWithNewOp( op, op.getValue(), subViewOp.getSource(), sourceIndices, diff --git a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp index 8590c56a49d4..44ccf05ff267 100644 --- a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp +++ b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp @@ -73,7 +73,7 @@ void mlir::sparse_tensor::buildSparseCompiler( pm.addPass(memref::createExpandStridedMetadataPass()); pm.addPass(createLowerAffinePass()); pm.addPass(createConvertVectorToLLVMPass(options.lowerVectorToLLVMOptions())); - pm.addPass(createMemRefToLLVMConversionPass()); + pm.addPass(createFinalizeMemRefToLLVMConversionPass()); pm.addNestedPass(createConvertComplexToStandardPass()); pm.addNestedPass(mlir::arith::createArithExpandOpsPass()); pm.addNestedPass(createConvertMathToLLVMPass()); diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 0981b3b1cbe2..cfc729d47eda 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -377,6 +377,9 @@ class AsmPrinter::Impl { /// dialect. void printResourceHandle(const AsmDialectResourceHandle &resource); + const void *getCurrentTypeOrAttrAlias() const; + void setCurrentTypeOrAttrAlias(const void *opaque); + void printAffineMap(AffineMap map); void printAffineExpr(AffineExpr expr, @@ -478,6 +481,10 @@ class SymbolAlias { bool isType : 1; /// A flag indicating whether this alias may be deferred or not. bool isDeferrable : 1; + +public: + /// Used to avoid printing incomplete aliases for recursive types. + bool isPrinted = false; }; /// This class represents a utility that initializes the set of attribute and @@ -753,6 +760,8 @@ class DummyAliasOperationPrinter : private OpAsmPrinter { void printSuccessorAndUseList(Block *, ValueRange) override {} void shadowRegionArgs(Region &, ValueRange) override {} + const void *getCurrentTypeOrAttrAlias() const override { return nullptr; } + /// The printer flags to use when determining potential aliases. const OpPrintingFlags &printerFlags; @@ -890,6 +899,8 @@ class DummyAliasDialectAsmPrinter : public DialectAsmPrinter { void printSymbolName(StringRef) override {} void printResourceHandle(const AsmDialectResourceHandle &) override {} + const void *getCurrentTypeOrAttrAlias() const override { return nullptr; } + /// The initializer to use when identifying aliases. AliasInitializer &initializer; @@ -1028,6 +1039,12 @@ std::pair AliasInitializer::visitImpl( void AliasInitializer::markAliasNonDeferrable(size_t aliasIndex) { auto it = std::next(aliases.begin(), aliasIndex); + + // If already marked non-deferrable stop the recursion. + // All children should already be marked non-deferrable as well. + if (!it->second.canBeDeferred) + return; + it->second.canBeDeferred = false; // Propagate the non-deferrable flag to any child aliases. @@ -1105,6 +1122,9 @@ class AliasState { /// An allocator used for alias names. llvm::BumpPtrAllocator aliasAllocator; + +public: + const void *currentTypeOrAttrAlias = nullptr; }; } // namespace @@ -1127,6 +1147,8 @@ LogicalResult AliasState::getAlias(Type ty, raw_ostream &os) const { auto it = attrTypeToAlias.find(ty.getAsOpaquePointer()); if (it == attrTypeToAlias.end()) return failure(); + if (!it->second.isPrinted) + return failure(); it->second.print(os); return success(); @@ -1142,13 +1164,16 @@ void AliasState::printAliases(AsmPrinter::Impl &p, NewLineCounter &newLine, alias.print(p.getStream()); p.getStream() << " = "; + currentTypeOrAttrAlias = opaqueSymbol; + auto guard = llvm::make_scope_exit([&]() { + currentTypeOrAttrAlias = nullptr; + }); + if (alias.isTypeAlias()) { // TODO: Support nested aliases in mutable types. Type type = Type::getFromOpaquePointer(opaqueSymbol); - if (type.hasTrait()) - p.getStream() << type; - else - p.printTypeImpl(type); + p.printTypeImpl(type); + alias.isPrinted = true; } else { // TODO: Support nested aliases in mutable attributes. Attribute attr = Attribute::getFromOpaquePointer(opaqueSymbol); @@ -2610,6 +2635,14 @@ void AsmPrinter::Impl::printHexString(ArrayRef data) { printHexString(StringRef(data.data(), data.size())); } +const void *AsmPrinter::Impl::getCurrentTypeOrAttrAlias() const { + return state.getAliasState().currentTypeOrAttrAlias; +} + +void AsmPrinter::Impl::setCurrentTypeOrAttrAlias(const void *opaque) { + state.getAliasState().currentTypeOrAttrAlias = opaque; +} + //===--------------------------------------------------------------------===// // AsmPrinter //===--------------------------------------------------------------------===// @@ -2668,6 +2701,11 @@ void AsmPrinter::printResourceHandle(const AsmDialectResourceHandle &resource) { impl->printResourceHandle(resource); } +const void *AsmPrinter::getCurrentTypeOrAttrAlias() const { + assert(impl && "expected AsmPrinter::getCurrentTypeOrAttrAlias to be overriden"); + return impl->getCurrentTypeOrAttrAlias(); +} + //===----------------------------------------------------------------------===// // Affine expressions and maps //===----------------------------------------------------------------------===// diff --git a/mlir/lib/IR/Dominance.cpp b/mlir/lib/IR/Dominance.cpp index beb7f7bfeedf..aabf9093892b 100644 --- a/mlir/lib/IR/Dominance.cpp +++ b/mlir/lib/IR/Dominance.cpp @@ -20,9 +20,16 @@ using namespace mlir; using namespace mlir::detail; -template class llvm::DominatorTreeBase; -template class llvm::DominatorTreeBase; -template class llvm::DomTreeNodeBase; +template class llvm::DominatorTreeOnView; +template class llvm::DominatorTreeOnView; +template class llvm::DomTreeNodeOnView; + +// Explicit instantiation for the `llvm::DomTreeBuilder::DeleteEdge` method, +template void llvm::DomTreeBuilder::DeleteEdge( + DomTreeOnView &DT, Block *From, Block *To); + +template void llvm::DomTreeBuilder::DeleteEdge( + PostDomTreeOnView &DT, Block *From, Block *To); //===----------------------------------------------------------------------===// // DominanceInfoBase diff --git a/mlir/lib/Parser/Parser.cpp b/mlir/lib/Parser/Parser.cpp index 57dd3eeb2714..b2fa463d94d1 100644 --- a/mlir/lib/Parser/Parser.cpp +++ b/mlir/lib/Parser/Parser.cpp @@ -90,7 +90,9 @@ LogicalResult mlir::parseSourceFile( LogicalResult mlir::parseSourceString(llvm::StringRef sourceStr, Block *block, const ParserConfig &config, LocationAttr *sourceFileLoc) { - auto memBuffer = llvm::MemoryBuffer::getMemBuffer(sourceStr); + auto memBuffer = + llvm::MemoryBuffer::getMemBuffer(sourceStr, "", + /*RequiresNullTerminator=*/false); if (!memBuffer) return failure(); diff --git a/mlir/lib/Target/LLVMIR/AttrKindDetail.h b/mlir/lib/Target/LLVMIR/AttrKindDetail.h new file mode 100644 index 000000000000..7f81777886f5 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/AttrKindDetail.h @@ -0,0 +1,66 @@ +//===- AttrKindDetail.h - AttrKind conversion details -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef ATTRKINDDETAIL_H_ +#define ATTRKINDDETAIL_H_ + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "llvm/IR/Attributes.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// Returns a list of pairs that each hold a mapping from LLVM attribute kinds +/// to their corresponding string name in LLVM IR dialect. +static llvm::ArrayRef> +getAttrKindToNameMapping() { + using ElemTy = std::pair; + // Mapping from llvm attribute kinds to their corresponding MLIR name. + static const llvm::SmallVector kindNamePairs = { + {llvm::Attribute::AttrKind::Alignment, LLVMDialect::getAlignAttrName()}, + {llvm::Attribute::AttrKind::AllocAlign, + LLVMDialect::getAllocAlignAttrName()}, + {llvm::Attribute::AttrKind::AllocatedPointer, + LLVMDialect::getAllocatedPointerAttrName()}, + {llvm::Attribute::AttrKind::ByVal, LLVMDialect::getByValAttrName()}, + {llvm::Attribute::AttrKind::ByRef, LLVMDialect::getByRefAttrName()}, + {llvm::Attribute::AttrKind::NoUndef, LLVMDialect::getNoUndefAttrName()}, + {llvm::Attribute::AttrKind::Dereferenceable, + LLVMDialect::getDereferenceableAttrName()}, + {llvm::Attribute::AttrKind::DereferenceableOrNull, + LLVMDialect::getDereferenceableOrNullAttrName()}, + {llvm::Attribute::AttrKind::InAlloca, LLVMDialect::getInAllocaAttrName()}, + {llvm::Attribute::AttrKind::InReg, LLVMDialect::getInRegAttrName()}, + {llvm::Attribute::AttrKind::Nest, LLVMDialect::getNestAttrName()}, + {llvm::Attribute::AttrKind::NoAlias, LLVMDialect::getNoAliasAttrName()}, + {llvm::Attribute::AttrKind::NoCapture, + LLVMDialect::getNoCaptureAttrName()}, + {llvm::Attribute::AttrKind::NoFree, LLVMDialect::getNoFreeAttrName()}, + {llvm::Attribute::AttrKind::NonNull, LLVMDialect::getNonNullAttrName()}, + {llvm::Attribute::AttrKind::Preallocated, + LLVMDialect::getPreallocatedAttrName()}, + {llvm::Attribute::AttrKind::ReadOnly, LLVMDialect::getReadonlyAttrName()}, + {llvm::Attribute::AttrKind::ReadNone, LLVMDialect::getReadnoneAttrName()}, + {llvm::Attribute::AttrKind::Returned, LLVMDialect::getReturnedAttrName()}, + {llvm::Attribute::AttrKind::SExt, LLVMDialect::getSExtAttrName()}, + {llvm::Attribute::AttrKind::StackAlignment, + LLVMDialect::getStackAlignmentAttrName()}, + {llvm::Attribute::AttrKind::StructRet, + LLVMDialect::getStructRetAttrName()}, + {llvm::Attribute::AttrKind::WriteOnly, + LLVMDialect::getWriteOnlyAttrName()}, + {llvm::Attribute::AttrKind::ZExt, LLVMDialect::getZExtAttrName()}}; + return kindNamePairs; +} + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#endif // ATTRKINDDETAIL_H_ diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt index 97577c036a22..c8e7797c7612 100644 --- a/mlir/lib/Target/LLVMIR/CMakeLists.txt +++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt @@ -5,6 +5,8 @@ set(LLVM_OPTIONAL_SOURCES ConvertToLLVMIR.cpp DebugTranslation.cpp DebugImporter.cpp + LoopAnnotationImporter.cpp + LoopAnnotationTranslation.cpp ModuleTranslation.cpp ModuleImport.cpp TypeToLLVM.cpp @@ -14,6 +16,7 @@ set(LLVM_OPTIONAL_SOURCES add_mlir_translation_library(MLIRTargetLLVMIRExport DebugTranslation.cpp + LoopAnnotationTranslation.cpp ModuleTranslation.cpp TypeToLLVM.cpp @@ -53,6 +56,7 @@ add_mlir_translation_library(MLIRToLLVMIRTranslationRegistration add_mlir_translation_library(MLIRTargetLLVMIRImport DebugImporter.cpp + LoopAnnotationImporter.cpp ModuleImport.cpp TypeFromLLVM.cpp diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp index 2e05f01119a0..bdfa6602915b 100644 --- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp +++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp @@ -17,6 +17,7 @@ #include "mlir/Target/LLVMIR/Import.h" #include "mlir/Tools/mlir-translate/Translation.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Verifier.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Support/SourceMgr.h" @@ -40,6 +41,8 @@ void registerFromLLVMIRTranslation() { emitError(UnknownLoc::get(context)) << errStream.str(); return {}; } + if (llvm::verifyModule(*llvmModule, &llvm::errs())) + return nullptr; return translateLLVMIRToModule(std::move(llvmModule), context); }, [](DialectRegistry ®istry) { diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp index cbb6754ea44d..cff8357457cb 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp +++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp @@ -47,7 +47,7 @@ DICompileUnitAttr DebugImporter::translateImpl(llvm::DICompileUnit *node) { symbolizeDIEmissionKind(node->getEmissionKind()); return DICompileUnitAttr::get(context, node->getSourceLanguage(), translate(node->getFile()), - StringAttr::get(context, node->getProducer()), + getStringAttrOrNull(node->getRawProducer()), node->isOptimized(), emissionKind.value()); } @@ -66,7 +66,7 @@ DICompositeTypeAttr DebugImporter::translateImpl(llvm::DICompositeType *node) { if (llvm::is_contained(elements, nullptr)) elements.clear(); return DICompositeTypeAttr::get( - context, node->getTag(), StringAttr::get(context, node->getName()), + context, node->getTag(), getStringAttrOrNull(node->getRawName()), translate(node->getFile()), node->getLine(), translate(node->getScope()), translate(node->getBaseType()), flags.value_or(DIFlags::Zero), node->getSizeInBits(), node->getAlignInBits(), elements); @@ -78,8 +78,7 @@ DIDerivedTypeAttr DebugImporter::translateImpl(llvm::DIDerivedType *node) { if (node->getBaseType() && !baseType) return nullptr; return DIDerivedTypeAttr::get( - context, node->getTag(), - node->getRawName() ? StringAttr::get(context, node->getName()) : nullptr, + context, node->getTag(), getStringAttrOrNull(node->getRawName()), baseType, node->getSizeInBits(), node->getAlignInBits(), node->getOffsetInBits()); } @@ -103,7 +102,7 @@ DebugImporter::translateImpl(llvm::DILexicalBlockFile *node) { DILocalVariableAttr DebugImporter::translateImpl(llvm::DILocalVariable *node) { return DILocalVariableAttr::get(context, translate(node->getScope()), - StringAttr::get(context, node->getName()), + getStringAttrOrNull(node->getRawName()), translate(node->getFile()), node->getLine(), node->getArg(), node->getAlignInBits(), translate(node->getType())); @@ -113,6 +112,12 @@ DIScopeAttr DebugImporter::translateImpl(llvm::DIScope *node) { return cast(translate(static_cast(node))); } +DINamespaceAttr DebugImporter::translateImpl(llvm::DINamespace *node) { + return DINamespaceAttr::get(context, getStringAttrOrNull(node->getRawName()), + translate(node->getScope()), + node->getExportSymbols()); +} + DISubprogramAttr DebugImporter::translateImpl(llvm::DISubprogram *node) { std::optional subprogramFlags = symbolizeDISubprogramFlags(node->getSubprogram()->getSPFlags()); @@ -123,14 +128,12 @@ DISubprogramAttr DebugImporter::translateImpl(llvm::DISubprogram *node) { DISubroutineTypeAttr type = translate(node->getType()); if (node->getType() && !type) return nullptr; - return DISubprogramAttr::get( - context, translate(node->getUnit()), scope, - StringAttr::get(context, node->getName()), - node->getRawLinkageName() - ? StringAttr::get(context, node->getLinkageName()) - : nullptr, - translate(node->getFile()), node->getLine(), node->getScopeLine(), - subprogramFlags.value(), type); + return DISubprogramAttr::get(context, translate(node->getUnit()), scope, + getStringAttrOrNull(node->getRawName()), + getStringAttrOrNull(node->getRawLinkageName()), + translate(node->getFile()), node->getLine(), + node->getScopeLine(), subprogramFlags.value(), + type); } DISubrangeAttr DebugImporter::translateImpl(llvm::DISubrange *node) { @@ -151,11 +154,11 @@ DebugImporter::translateImpl(llvm::DISubroutineType *node) { SmallVector types; for (llvm::DIType *type : node->getTypeArray()) { if (!type) { - // A nullptr entry at the beginning of the subroutine types list models a - // void result type. Translate the nullptr to an explicit - // DIVoidResultTypeAttr since the attribute list cannot contain a nullptr - // entry. - types.push_back(DIVoidResultTypeAttr::get(context)); + // A nullptr entry may appear at the beginning or the end of the + // subroutine types list modeling either a void result type or the type of + // a variadic argument. Translate the nullptr to an explicit + // DINullTypeAttr since the attribute list cannot contain a nullptr entry. + types.push_back(DINullTypeAttr::get(context)); continue; } types.push_back(translate(type)); @@ -205,6 +208,8 @@ DINodeAttr DebugImporter::translate(llvm::DINode *node) { return translateImpl(casted); if (auto *casted = dyn_cast(node)) return translateImpl(casted); + if (auto *casted = dyn_cast(node)) + return translateImpl(casted); if (auto *casted = dyn_cast(node)) return translateImpl(casted); if (auto *casted = dyn_cast(node)) @@ -240,3 +245,9 @@ Location DebugImporter::translateLoc(llvm::DILocation *loc) { context); return result; } + +StringAttr DebugImporter::getStringAttrOrNull(llvm::MDString *stringNode) { + if (!stringNode) + return StringAttr(); + return StringAttr::get(context, stringNode->getString()); +} diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.h b/mlir/lib/Target/LLVMIR/DebugImporter.h index 7ae633df6e7b..e7f2332ff436 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.h +++ b/mlir/lib/Target/LLVMIR/DebugImporter.h @@ -62,10 +62,15 @@ class DebugImporter { DILocalVariableAttr translateImpl(llvm::DILocalVariable *node); DIScopeAttr translateImpl(llvm::DIScope *node); DISubprogramAttr translateImpl(llvm::DISubprogram *node); + DINamespaceAttr translateImpl(llvm::DINamespace *node); DISubrangeAttr translateImpl(llvm::DISubrange *node); DISubroutineTypeAttr translateImpl(llvm::DISubroutineType *node); DITypeAttr translateImpl(llvm::DIType *node); + /// Constructs a StringAttr from the MDString if it is non-null. Returns a + /// null attribute otherwise. + StringAttr getStringAttrOrNull(llvm::MDString *stringNode); + /// A mapping between LLVM debug metadata and the corresponding attribute. DenseMap nodeToAttr; diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp index fd80a773df8e..dae9930ea309 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp @@ -88,25 +88,35 @@ void DebugTranslation::translate(LLVMFuncOp func, llvm::Function &llvmFunc) { // Attributes //===----------------------------------------------------------------------===// -llvm::DIType *DebugTranslation::translateImpl(DIVoidResultTypeAttr attr) { - // A DIVoidResultTypeAttr at the beginning of the subroutine types list models - // a void result type. Translate the explicit DIVoidResultTypeAttr to a - // nullptr since LLVM IR metadata does not have an explicit void result type +llvm::DIType *DebugTranslation::translateImpl(DINullTypeAttr attr) { + // A DINullTypeAttr at the beginning of the subroutine types list models + // a void result type. If it is at the end, it models a variadic function. + // Translate the explicit DINullTypeAttr to a nullptr since LLVM IR metadata + // does not have an explicit void result type nor a variadic type // representation. return nullptr; } +llvm::MDString *DebugTranslation::getMDStringOrNull(StringAttr stringAttr) { + if (!stringAttr || stringAttr.getValue().empty()) + return nullptr; + return llvm::MDString::get(llvmCtx, stringAttr); +} + llvm::DIBasicType *DebugTranslation::translateImpl(DIBasicTypeAttr attr) { return llvm::DIBasicType::get( - llvmCtx, attr.getTag(), attr.getName(), attr.getSizeInBits(), + llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()), + attr.getSizeInBits(), /*AlignInBits=*/0, attr.getEncoding(), llvm::DINode::FlagZero); } llvm::DICompileUnit *DebugTranslation::translateImpl(DICompileUnitAttr attr) { llvm::DIBuilder builder(llvmModule); return builder.createCompileUnit( - attr.getSourceLanguage(), translate(attr.getFile()), attr.getProducer(), - attr.getIsOptimized(), /*Flags=*/"", /*RV=*/0); + attr.getSourceLanguage(), translate(attr.getFile()), + attr.getProducer() ? attr.getProducer().getValue() : "", + attr.getIsOptimized(), + /*Flags=*/"", /*RV=*/0); } llvm::DICompositeType * @@ -115,9 +125,10 @@ DebugTranslation::translateImpl(DICompositeTypeAttr attr) { for (auto member : attr.getElements()) elements.push_back(translate(member)); return llvm::DICompositeType::get( - llvmCtx, attr.getTag(), attr.getName(), translate(attr.getFile()), - attr.getLine(), translate(attr.getScope()), translate(attr.getBaseType()), - attr.getSizeInBits(), attr.getAlignInBits(), + llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()), + translate(attr.getFile()), attr.getLine(), translate(attr.getScope()), + translate(attr.getBaseType()), attr.getSizeInBits(), + attr.getAlignInBits(), /*OffsetInBits=*/0, /*Flags=*/static_cast(attr.getFlags()), llvm::MDNode::get(llvmCtx, elements), @@ -125,9 +136,6 @@ DebugTranslation::translateImpl(DICompositeTypeAttr attr) { } llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) { - auto getMDStringOrNull = [&](StringAttr attr) -> llvm::MDString * { - return attr ? llvm::MDString::get(llvmCtx, attr) : nullptr; - }; return llvm::DIDerivedType::get( llvmCtx, attr.getTag(), getMDStringOrNull(attr.getName()), /*File=*/nullptr, /*Line=*/0, @@ -137,7 +145,8 @@ llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) { } llvm::DIFile *DebugTranslation::translateImpl(DIFileAttr attr) { - return llvm::DIFile::get(llvmCtx, attr.getName(), attr.getDirectory()); + return llvm::DIFile::get(llvmCtx, getMDStringOrNull(attr.getName()), + getMDStringOrNull(attr.getDirectory())); } llvm::DILexicalBlock *DebugTranslation::translateImpl(DILexicalBlockAttr attr) { @@ -160,9 +169,9 @@ llvm::DILocalScope *DebugTranslation::translateImpl(DILocalScopeAttr attr) { llvm::DILocalVariable * DebugTranslation::translateImpl(DILocalVariableAttr attr) { return llvm::DILocalVariable::get( - llvmCtx, translate(attr.getScope()), - llvm::MDString::get(llvmCtx, attr.getName()), translate(attr.getFile()), - attr.getLine(), translate(attr.getType()), attr.getArg(), + llvmCtx, translate(attr.getScope()), getMDStringOrNull(attr.getName()), + translate(attr.getFile()), attr.getLine(), translate(attr.getType()), + attr.getArg(), /*Flags=*/llvm::DINode::FlagZero, attr.getAlignInBits(), /*Annotations=*/nullptr); } @@ -183,12 +192,9 @@ static llvm::DISubprogram *getSubprogram(bool isDistinct, Ts &&...args) { llvm::DISubprogram *DebugTranslation::translateImpl(DISubprogramAttr attr) { bool isDefinition = static_cast(attr.getSubprogramFlags() & LLVM::DISubprogramFlags::Definition); - auto getMDStringOrNull = [&](StringAttr attr) -> llvm::MDString * { - return attr ? llvm::MDString::get(llvmCtx, attr) : nullptr; - }; return getSubprogram( isDefinition, llvmCtx, translate(attr.getScope()), - llvm::MDString::get(llvmCtx, attr.getName()), + getMDStringOrNull(attr.getName()), getMDStringOrNull(attr.getLinkageName()), translate(attr.getFile()), attr.getLine(), translate(attr.getType()), attr.getScopeLine(), /*ContainingType=*/nullptr, /*VirtualIndex=*/0, @@ -197,6 +203,12 @@ llvm::DISubprogram *DebugTranslation::translateImpl(DISubprogramAttr attr) { translate(attr.getCompileUnit())); } +llvm::DINamespace *DebugTranslation::translateImpl(DINamespaceAttr attr) { + return llvm::DINamespace::get(llvmCtx, translate(attr.getScope()), + getMDStringOrNull(attr.getName()), + attr.getExportSymbols()); +} + llvm::DISubrange *DebugTranslation::translateImpl(DISubrangeAttr attr) { auto getMetadataOrNull = [&](IntegerAttr attr) -> llvm::Metadata * { if (!attr) @@ -234,10 +246,11 @@ llvm::DINode *DebugTranslation::translate(DINodeAttr attr) { llvm::DINode *node = TypeSwitch(attr) - .Case( + .Case( [&](auto attr) { return translateImpl(attr); }); attrToNode.insert({attr, node}); return node; diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.h b/mlir/lib/Target/LLVMIR/DebugTranslation.h index b62bd2bab4ce..3ea077a80bfa 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.h +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.h @@ -61,7 +61,7 @@ class DebugTranslation { llvm::DIFile *translateFile(StringRef fileName); /// Translate the given attribute to the corresponding llvm debug metadata. - llvm::DIType *translateImpl(DIVoidResultTypeAttr attr); + llvm::DIType *translateImpl(DINullTypeAttr attr); llvm::DIBasicType *translateImpl(DIBasicTypeAttr attr); llvm::DICompileUnit *translateImpl(DICompileUnitAttr attr); llvm::DICompositeType *translateImpl(DICompositeTypeAttr attr); @@ -73,10 +73,15 @@ class DebugTranslation { llvm::DILocalVariable *translateImpl(DILocalVariableAttr attr); llvm::DIScope *translateImpl(DIScopeAttr attr); llvm::DISubprogram *translateImpl(DISubprogramAttr attr); + llvm::DINamespace *translateImpl(DINamespaceAttr attr); llvm::DISubrange *translateImpl(DISubrangeAttr attr); llvm::DISubroutineType *translateImpl(DISubroutineTypeAttr attr); llvm::DIType *translateImpl(DITypeAttr attr); + /// Constructs a string metadata node from the string attribute. Returns + /// nullptr if `stringAttr` is null or contains and empty string. + llvm::MDString *getMDStringOrNull(StringAttr stringAttr); + /// A mapping between mlir location+scope and the corresponding llvm debug /// metadata. DenseMap, diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index 7d55eae86cdd..ed9722bd8d9e 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -71,20 +71,45 @@ static LogicalResult convertIntrinsicImpl(OpBuilder &odsBuilder, /// dialect attributes. static ArrayRef getSupportedMetadataImpl() { static const SmallVector convertibleMetadata = { - llvm::LLVMContext::MD_prof, // profiling metadata - llvm::LLVMContext::MD_tbaa}; + llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa, + llvm::LLVMContext::MD_access_group, llvm::LLVMContext::MD_loop, + llvm::LLVMContext::MD_noalias, llvm::LLVMContext::MD_alias_scope}; return convertibleMetadata; } -/// Attaches the given profiling metadata to the imported operation if a -/// conversion to an MLIR profiling attribute exists and succeeds. Returns -/// failure otherwise. -static LogicalResult setProfilingAttrs(OpBuilder &builder, llvm::MDNode *node, - Operation *op, - LLVM::ModuleImport &moduleImport) { - // Return success for empty metadata nodes since there is nothing to import. +namespace { +/// Helper class to attach metadata attributes to specific operation types. It +/// specializes TypeSwitch to take an Operation and return a LogicalResult. +template +struct AttributeSetter { + AttributeSetter(Operation *op) : op(op) {} + + /// Calls `attachFn` on the provided Operation if it has one of + /// the given operation types. Returns failure otherwise. + template + LogicalResult apply(CallableT &&attachFn) { + return llvm::TypeSwitch(op) + .Case([&attachFn](auto concreteOp) { + attachFn(concreteOp); + return success(); + }) + .Default([&](auto) { return failure(); }); + } + +private: + Operation *op; +}; +} // namespace + +/// Converts the given profiling metadata `node` to an MLIR profiling attribute +/// and attaches it to the imported operation if the translation succeeds. +/// Returns failure otherwise. +static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { + // Return failure for empty metadata nodes since there is nothing to import. if (!node->getNumOperands()) - return success(); + return failure(); auto *name = dyn_cast(node->getOperand(0)); if (!name) @@ -92,17 +117,21 @@ static LogicalResult setProfilingAttrs(OpBuilder &builder, llvm::MDNode *node, // Handle function entry count metadata. if (name->getString().equals("function_entry_count")) { + // TODO support function entry count metadata with GUID fields. if (node->getNumOperands() != 2) return failure(); llvm::ConstantInt *entryCount = - llvm::mdconst::extract(node->getOperand(1)); + llvm::mdconst::dyn_extract(node->getOperand(1)); + if (!entryCount) + return failure(); if (auto funcOp = dyn_cast(op)) { funcOp.setFunctionEntryCount(entryCount->getZExtValue()); return success(); } - return failure(); + return op->emitWarning() + << "expected function_entry_count to be attached to a function"; } if (!name->getString().equals("branch_weights")) @@ -113,32 +142,108 @@ static LogicalResult setProfilingAttrs(OpBuilder &builder, llvm::MDNode *node, branchWeights.reserve(node->getNumOperands() - 1); for (unsigned i = 1, e = node->getNumOperands(); i != e; ++i) { llvm::ConstantInt *branchWeight = - llvm::mdconst::extract(node->getOperand(i)); + llvm::mdconst::dyn_extract(node->getOperand(i)); + if (!branchWeight) + return failure(); branchWeights.push_back(branchWeight->getZExtValue()); } - // Attach the branch weights to the operations that support it. - return llvm::TypeSwitch(op) - .Case([&](auto branchWeightOp) { + return AttributeSetter(op).apply( + [&](auto branchWeightOp) { branchWeightOp.setBranchWeightsAttr( builder.getI32VectorAttr(branchWeights)); + }); +} + +/// Searches the symbol reference pointing to the metadata operation that +/// maps to the given TBAA metadata `node` and attaches it to the imported +/// operation if the lookup succeeds. Returns failure otherwise. +static LogicalResult setTBAAAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { + SymbolRefAttr tbaaTagSym = moduleImport.lookupTBAAAttr(node); + if (!tbaaTagSym) + return failure(); + + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setTbaaAttr(ArrayAttr::get(memOp.getContext(), tbaaTagSym)); + }); +} + +/// Looks up all the symbol references pointing to the access group operations +/// that map to the access group nodes starting from the access group metadata +/// `node`, and attaches all of them to the imported operation if the lookups +/// succeed. Returns failure otherwise. +static LogicalResult setAccessGroupsAttr(const llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { + FailureOr> accessGroups = + moduleImport.lookupAccessGroupAttrs(node); + if (failed(accessGroups)) + return failure(); + + SmallVector accessGroupAttrs(accessGroups->begin(), + accessGroups->end()); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setAccessGroupsAttr( + ArrayAttr::get(memOp.getContext(), accessGroupAttrs)); + }); +} + +/// Converts the given loop metadata node to an MLIR loop annotation attribute +/// and attaches it to the imported operation if the translation succeeds. +/// Returns failure otherwise. +static LogicalResult setLoopAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { + LoopAnnotationAttr attr = + moduleImport.translateLoopAnnotationAttr(node, op->getLoc()); + if (!attr) + return failure(); + + return TypeSwitch(op) + .Case([&](auto branchOp) { + branchOp.setLoopAnnotationAttr(attr); return success(); }) .Default([](auto) { return failure(); }); } -/// Attaches the given TBAA metadata `node` to the imported operation. -/// Returns success, if the metadata has been converted and the attachment -/// succeeds, failure - otherwise. -static LogicalResult setTBAAAttrs(const llvm::MDNode *node, Operation *op, - LLVM::ModuleImport &moduleImport) { - SymbolRefAttr tbaaTagSym = moduleImport.lookupTBAAAttr(node); - if (!tbaaTagSym) +/// Looks up all the symbol references pointing to the alias scope operations +/// that map to the alias scope nodes starting from the alias scope metadata +/// `node`, and attaches all of them to the imported operation if the lookups +/// succeed. Returns failure otherwise. +static LogicalResult setAliasScopesAttr(const llvm::MDNode *node, Operation *op, + LLVM::ModuleImport &moduleImport) { + FailureOr> aliasScopes = + moduleImport.lookupAliasScopeAttrs(node); + if (failed(aliasScopes)) return failure(); - op->setAttr(LLVMDialect::getTBAAAttrName(), - ArrayAttr::get(op->getContext(), tbaaTagSym)); - return success(); + SmallVector aliasScopeAttrs(aliasScopes->begin(), + aliasScopes->end()); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setAliasScopesAttr( + ArrayAttr::get(memOp.getContext(), aliasScopeAttrs)); + }); +} + +/// Looks up all the symbol references pointing to the alias scope operations +/// that map to the alias scope nodes starting from the noalias metadata `node`, +/// and attaches all of them to the imported operation if the lookups succeed. +/// Returns failure otherwise. +static LogicalResult setNoaliasScopesAttr(const llvm::MDNode *node, + Operation *op, + LLVM::ModuleImport &moduleImport) { + FailureOr> noaliasScopes = + moduleImport.lookupAliasScopeAttrs(node); + if (failed(noaliasScopes)) + return failure(); + + SmallVector noaliasScopeAttrs(noaliasScopes->begin(), + noaliasScopes->end()); + return AttributeSetter(op).apply([&](auto memOp) { + memOp.setNoaliasScopesAttr( + ArrayAttr::get(memOp.getContext(), noaliasScopeAttrs)); + }); } namespace { @@ -164,9 +269,17 @@ class LLVMDialectLLVMIRImportInterface : public LLVMImportDialectInterface { LLVM::ModuleImport &moduleImport) const final { // Call metadata specific handlers. if (kind == llvm::LLVMContext::MD_prof) - return setProfilingAttrs(builder, node, op, moduleImport); + return setProfilingAttr(builder, node, op, moduleImport); if (kind == llvm::LLVMContext::MD_tbaa) - return setTBAAAttrs(node, op, moduleImport); + return setTBAAAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_access_group) + return setAccessGroupsAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_loop) + return setLoopAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_alias_scope) + return setAliasScopesAttr(node, op, moduleImport); + if (kind == llvm::LLVMContext::MD_noalias) + return setNoaliasScopesAttr(node, op, moduleImport); // A handler for a supported metadata kind is missing. llvm_unreachable("unknown metadata type"); diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp index 7f44db50f820..5c98f922b22e 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -28,123 +28,6 @@ using mlir::LLVM::detail::getLLVMConstant; #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc" -/// Convert MLIR integer comparison predicate to LLVM IR comparison predicate. -static llvm::CmpInst::Predicate getLLVMCmpPredicate(ICmpPredicate p) { - switch (p) { - case LLVM::ICmpPredicate::eq: - return llvm::CmpInst::Predicate::ICMP_EQ; - case LLVM::ICmpPredicate::ne: - return llvm::CmpInst::Predicate::ICMP_NE; - case LLVM::ICmpPredicate::slt: - return llvm::CmpInst::Predicate::ICMP_SLT; - case LLVM::ICmpPredicate::sle: - return llvm::CmpInst::Predicate::ICMP_SLE; - case LLVM::ICmpPredicate::sgt: - return llvm::CmpInst::Predicate::ICMP_SGT; - case LLVM::ICmpPredicate::sge: - return llvm::CmpInst::Predicate::ICMP_SGE; - case LLVM::ICmpPredicate::ult: - return llvm::CmpInst::Predicate::ICMP_ULT; - case LLVM::ICmpPredicate::ule: - return llvm::CmpInst::Predicate::ICMP_ULE; - case LLVM::ICmpPredicate::ugt: - return llvm::CmpInst::Predicate::ICMP_UGT; - case LLVM::ICmpPredicate::uge: - return llvm::CmpInst::Predicate::ICMP_UGE; - } - llvm_unreachable("incorrect comparison predicate"); -} - -static llvm::CmpInst::Predicate getLLVMCmpPredicate(FCmpPredicate p) { - switch (p) { - case LLVM::FCmpPredicate::_false: - return llvm::CmpInst::Predicate::FCMP_FALSE; - case LLVM::FCmpPredicate::oeq: - return llvm::CmpInst::Predicate::FCMP_OEQ; - case LLVM::FCmpPredicate::ogt: - return llvm::CmpInst::Predicate::FCMP_OGT; - case LLVM::FCmpPredicate::oge: - return llvm::CmpInst::Predicate::FCMP_OGE; - case LLVM::FCmpPredicate::olt: - return llvm::CmpInst::Predicate::FCMP_OLT; - case LLVM::FCmpPredicate::ole: - return llvm::CmpInst::Predicate::FCMP_OLE; - case LLVM::FCmpPredicate::one: - return llvm::CmpInst::Predicate::FCMP_ONE; - case LLVM::FCmpPredicate::ord: - return llvm::CmpInst::Predicate::FCMP_ORD; - case LLVM::FCmpPredicate::ueq: - return llvm::CmpInst::Predicate::FCMP_UEQ; - case LLVM::FCmpPredicate::ugt: - return llvm::CmpInst::Predicate::FCMP_UGT; - case LLVM::FCmpPredicate::uge: - return llvm::CmpInst::Predicate::FCMP_UGE; - case LLVM::FCmpPredicate::ult: - return llvm::CmpInst::Predicate::FCMP_ULT; - case LLVM::FCmpPredicate::ule: - return llvm::CmpInst::Predicate::FCMP_ULE; - case LLVM::FCmpPredicate::une: - return llvm::CmpInst::Predicate::FCMP_UNE; - case LLVM::FCmpPredicate::uno: - return llvm::CmpInst::Predicate::FCMP_UNO; - case LLVM::FCmpPredicate::_true: - return llvm::CmpInst::Predicate::FCMP_TRUE; - } - llvm_unreachable("incorrect comparison predicate"); -} - -static llvm::AtomicRMWInst::BinOp getLLVMAtomicBinOp(AtomicBinOp op) { - switch (op) { - case LLVM::AtomicBinOp::xchg: - return llvm::AtomicRMWInst::BinOp::Xchg; - case LLVM::AtomicBinOp::add: - return llvm::AtomicRMWInst::BinOp::Add; - case LLVM::AtomicBinOp::sub: - return llvm::AtomicRMWInst::BinOp::Sub; - case LLVM::AtomicBinOp::_and: - return llvm::AtomicRMWInst::BinOp::And; - case LLVM::AtomicBinOp::nand: - return llvm::AtomicRMWInst::BinOp::Nand; - case LLVM::AtomicBinOp::_or: - return llvm::AtomicRMWInst::BinOp::Or; - case LLVM::AtomicBinOp::_xor: - return llvm::AtomicRMWInst::BinOp::Xor; - case LLVM::AtomicBinOp::max: - return llvm::AtomicRMWInst::BinOp::Max; - case LLVM::AtomicBinOp::min: - return llvm::AtomicRMWInst::BinOp::Min; - case LLVM::AtomicBinOp::umax: - return llvm::AtomicRMWInst::BinOp::UMax; - case LLVM::AtomicBinOp::umin: - return llvm::AtomicRMWInst::BinOp::UMin; - case LLVM::AtomicBinOp::fadd: - return llvm::AtomicRMWInst::BinOp::FAdd; - case LLVM::AtomicBinOp::fsub: - return llvm::AtomicRMWInst::BinOp::FSub; - } - llvm_unreachable("incorrect atomic binary operator"); -} - -static llvm::AtomicOrdering getLLVMAtomicOrdering(AtomicOrdering ordering) { - switch (ordering) { - case LLVM::AtomicOrdering::not_atomic: - return llvm::AtomicOrdering::NotAtomic; - case LLVM::AtomicOrdering::unordered: - return llvm::AtomicOrdering::Unordered; - case LLVM::AtomicOrdering::monotonic: - return llvm::AtomicOrdering::Monotonic; - case LLVM::AtomicOrdering::acquire: - return llvm::AtomicOrdering::Acquire; - case LLVM::AtomicOrdering::release: - return llvm::AtomicOrdering::Release; - case LLVM::AtomicOrdering::acq_rel: - return llvm::AtomicOrdering::AcquireRelease; - case LLVM::AtomicOrdering::seq_cst: - return llvm::AtomicOrdering::SequentiallyConsistent; - } - llvm_unreachable("incorrect atomic ordering"); -} - static llvm::FastMathFlags getFastmathFlags(FastmathFlagsInterface &op) { using llvmFMF = llvm::FastMathFlags; using FuncT = void (llvmFMF::*)(bool); @@ -167,90 +50,6 @@ static llvm::FastMathFlags getFastmathFlags(FastmathFlagsInterface &op) { return ret; } -/// Returns an LLVM metadata node corresponding to a loop option. This metadata -/// is attached to an llvm.loop node. -static llvm::MDNode *getLoopOptionMetadata(llvm::LLVMContext &ctx, - LoopOptionCase option, - int64_t value) { - StringRef name; - llvm::Constant *cstValue = nullptr; - switch (option) { - case LoopOptionCase::disable_licm: - name = "llvm.licm.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::disable_unroll: - name = "llvm.loop.unroll.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::interleave_count: - name = "llvm.loop.interleave.count"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; - case LoopOptionCase::disable_pipeline: - name = "llvm.loop.pipeline.disable"; - cstValue = llvm::ConstantInt::getBool(ctx, value); - break; - case LoopOptionCase::pipeline_initiation_interval: - name = "llvm.loop.pipeline.initiationinterval"; - cstValue = llvm::ConstantInt::get( - llvm::IntegerType::get(ctx, /*NumBits=*/32), value); - break; - } - return llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), - llvm::ConstantAsMetadata::get(cstValue)}); -} - -static void setLoopMetadata(Operation &opInst, llvm::Instruction &llvmInst, - llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation) { - if (Attribute attr = opInst.getAttr(LLVMDialect::getLoopAttrName())) { - llvm::Module *module = builder.GetInsertBlock()->getModule(); - llvm::MDNode *loopMD = moduleTranslation.lookupLoopOptionsMetadata(attr); - if (!loopMD) { - llvm::LLVMContext &ctx = module->getContext(); - - SmallVector loopOptions; - // Reserve operand 0 for loop id self reference. - auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); - loopOptions.push_back(dummy.get()); - - auto loopAttr = attr.cast(); - auto parallelAccessGroup = - loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName()); - if (parallelAccessGroup) { - SmallVector parallelAccess; - parallelAccess.push_back( - llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); - for (SymbolRefAttr accessGroupRef : parallelAccessGroup->getValue() - .cast() - .getAsRange()) - parallelAccess.push_back( - moduleTranslation.getAccessGroup(opInst, accessGroupRef)); - loopOptions.push_back(llvm::MDNode::get(ctx, parallelAccess)); - } - - if (auto loopOptionsAttr = loopAttr.getAs( - LLVMDialect::getLoopOptionsAttrName())) { - for (auto option : loopOptionsAttr.getOptions()) - loopOptions.push_back( - getLoopOptionMetadata(ctx, option.first, option.second)); - } - - // Create loop options and set the first operand to itself. - loopMD = llvm::MDNode::get(ctx, loopOptions); - loopMD->replaceOperandWith(0, loopMD); - - // Store a map from this Attribute to the LLVM metadata in case we - // encounter it again. - moduleTranslation.mapLoopOptionsMetadata(attr, loopMD); - } - - llvmInst.setMetadata(module->getMDKindID("llvm.loop"), loopMD); - } -} - /// Convert the value of a DenseI64ArrayAttr to a vector of unsigned indices. static SmallVector extractPosition(ArrayRef indices) { SmallVector position; @@ -348,6 +147,22 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, #include "mlir/Dialect/LLVMIR/LLVMConversions.inc" #include "mlir/Dialect/LLVMIR/LLVMIntrinsicConversions.inc" + // Helper function to reconstruct the function type for an indirect call given + // the result and argument types. The function cannot reconstruct the type of + // variadic functions since the call operation does not carry enough + // information to distinguish normal and variadic arguments. Supporting + // indirect variadic calls requires an additional type attribute on the call + // operation that stores the LLVM function type of the callee. + // TODO: Support indirect calls to variadic function pointers. + auto getCalleeFunctionType = [&](TypeRange resultTypes, ValueRange args) { + Type resultType = resultTypes.empty() + ? LLVMVoidType::get(opInst.getContext()) + : resultTypes.front(); + return llvm::cast(moduleTranslation.convertType( + LLVMFunctionType::get(opInst.getContext(), resultType, + llvm::to_vector(args.getTypes()), false))); + }; + // Emit function calls. If the "callee" attribute is present, this is a // direct function call and we also need to look up the remapped function // itself. Otherwise, this is an indirect call and the callee is the first @@ -360,12 +175,9 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, call = builder.CreateCall( moduleTranslation.lookupFunction(attr.getValue()), operandsRef); } else { - auto calleeType = - callOp->getOperands().front().getType().cast(); - auto *calleeFunctionType = cast( - moduleTranslation.convertType(calleeType.getElementType())); - call = builder.CreateCall(calleeFunctionType, operandsRef.front(), - operandsRef.drop_front()); + call = builder.CreateCall(getCalleeFunctionType(callOp.getResultTypes(), + callOp.getArgOperands()), + operandsRef.front(), operandsRef.drop_front()); } llvm::MDNode *branchWeights = convertBranchWeights(callOp.getBranchWeights(), moduleTranslation); @@ -449,12 +261,9 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, moduleTranslation.lookupBlock(invOp.getSuccessor(0)), moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef); } else { - auto calleeType = - invOp.getCalleeOperands().front().getType().cast(); - auto *calleeFunctionType = cast( - moduleTranslation.convertType(calleeType.getElementType())); result = builder.CreateInvoke( - calleeFunctionType, operandsRef.front(), + getCalleeFunctionType(invOp.getResultTypes(), invOp.getArgOperands()), + operandsRef.front(), moduleTranslation.lookupBlock(invOp.getSuccessor(0)), moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef.drop_front()); @@ -495,7 +304,7 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, llvm::BranchInst *branch = builder.CreateBr(moduleTranslation.lookupBlock(brOp.getSuccessor())); moduleTranslation.mapBranch(&opInst, branch); - setLoopMetadata(opInst, *branch, builder, moduleTranslation); + moduleTranslation.setLoopMetadata(&opInst, branch); return success(); } if (auto condbrOp = dyn_cast(opInst)) { @@ -506,7 +315,7 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder, moduleTranslation.lookupBlock(condbrOp.getSuccessor(0)), moduleTranslation.lookupBlock(condbrOp.getSuccessor(1)), branchWeights); moduleTranslation.mapBranch(&opInst, branch); - setLoopMetadata(opInst, *branch, builder, moduleTranslation); + moduleTranslation.setLoopMetadata(&opInst, branch); return success(); } if (auto switchOp = dyn_cast(opInst)) { diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp new file mode 100644 index 000000000000..c086f4c318e7 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp @@ -0,0 +1,507 @@ +//===- LoopAnnotationImporter.cpp - Loop annotation import ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "LoopAnnotationImporter.h" +#include "llvm/IR/Constants.h" + +using namespace mlir; +using namespace mlir::LLVM; +using namespace mlir::LLVM::detail; + +namespace { +/// Helper class that keeps the state of one metadata to attribute conversion. +struct LoopMetadataConversion { + LoopMetadataConversion(const llvm::MDNode *node, Location loc, + LoopAnnotationImporter &loopAnnotationImporter) + : node(node), loc(loc), loopAnnotationImporter(loopAnnotationImporter), + ctx(loc->getContext()){}; + /// Converts this structs loop metadata node into a LoopAnnotationAttr. + LoopAnnotationAttr convert(); + + LogicalResult initPropertyMap(); + + /// Helper function to get and erase a property. + const llvm::MDNode *lookupAndEraseProperty(StringRef name); + + /// Helper functions to lookup and convert MDNodes into a specifc attribute + /// kind. These functions return null-attributes if there is no node with the + /// specified name, or failure, if the node is ill-formatted. + FailureOr lookupUnitNode(StringRef name); + FailureOr lookupBoolNode(StringRef name, bool negated = false); + FailureOr lookupIntNodeAsBoolAttr(StringRef name); + FailureOr lookupIntNode(StringRef name); + FailureOr lookupMDNode(StringRef name); + FailureOr> lookupMDNodes(StringRef name); + FailureOr lookupFollowupNode(StringRef name); + FailureOr lookupBooleanUnitNode(StringRef enableName, + StringRef disableName, + bool negated = false); + + /// Conversion functions for sub-attributes. + FailureOr convertVectorizeAttr(); + FailureOr convertInterleaveAttr(); + FailureOr convertUnrollAttr(); + FailureOr convertUnrollAndJamAttr(); + FailureOr convertLICMAttr(); + FailureOr convertDistributeAttr(); + FailureOr convertPipelineAttr(); + FailureOr convertPeeledAttr(); + FailureOr convertUnswitchAttr(); + FailureOr> convertParallelAccesses(); + + llvm::StringMap propertyMap; + const llvm::MDNode *node; + Location loc; + LoopAnnotationImporter &loopAnnotationImporter; + MLIRContext *ctx; +}; +} // namespace + +LogicalResult LoopMetadataConversion::initPropertyMap() { + // Check if it's a valid node. + if (node->getNumOperands() == 0 || + dyn_cast(node->getOperand(0)) != node) + return emitWarning(loc) << "invalid loop node"; + + for (const llvm::MDOperand &operand : llvm::drop_begin(node->operands())) { + // Skip over DILocations. + if (isa(operand)) + continue; + + auto *property = dyn_cast(operand); + if (!property) + return emitWarning(loc) << "expected all loop properties to be either " + "debug locations or metadata nodes"; + + if (property->getNumOperands() == 0) + return emitWarning(loc) << "cannot import empty loop property"; + + auto *nameNode = dyn_cast(property->getOperand(0)); + if (!nameNode) + return emitWarning(loc) << "cannot import loop property without a name"; + StringRef name = nameNode->getString(); + + bool succ = propertyMap.try_emplace(name, property).second; + if (!succ) + return emitWarning(loc) + << "cannot import loop properties with duplicated names " << name; + } + + return success(); +} + +const llvm::MDNode * +LoopMetadataConversion::lookupAndEraseProperty(StringRef name) { + auto it = propertyMap.find(name); + if (it == propertyMap.end()) + return nullptr; + const llvm::MDNode *property = it->getValue(); + propertyMap.erase(it); + return property; +} + +FailureOr LoopMetadataConversion::lookupUnitNode(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return BoolAttr(nullptr); + + if (property->getNumOperands() != 1) + return emitWarning(loc) + << "expected metadata node " << name << " to hold no value"; + + return BoolAttr::get(ctx, true); +} + +FailureOr LoopMetadataConversion::lookupBooleanUnitNode( + StringRef enableName, StringRef disableName, bool negated) { + auto enable = lookupUnitNode(enableName); + auto disable = lookupUnitNode(disableName); + if (failed(enable) || failed(disable)) + return failure(); + + if (*enable && *disable) + return emitWarning(loc) + << "expected metadata nodes " << enableName << " and " << disableName + << " to be mutually exclusive."; + + if (*enable) + return BoolAttr::get(ctx, !negated); + + if (*disable) + return BoolAttr::get(ctx, negated); + return BoolAttr(nullptr); +} + +FailureOr LoopMetadataConversion::lookupBoolNode(StringRef name, + bool negated) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return BoolAttr(nullptr); + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold a boolean value"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + llvm::ConstantInt *val = + llvm::mdconst::dyn_extract(property->getOperand(1)); + if (!val || val->getBitWidth() != 1) + return emitNodeWarning(); + + return BoolAttr::get(ctx, val->getValue().getLimitedValue(1) ^ negated); +} + +FailureOr +LoopMetadataConversion::lookupIntNodeAsBoolAttr(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return BoolAttr(nullptr); + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold an integer value"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + llvm::ConstantInt *val = + llvm::mdconst::dyn_extract(property->getOperand(1)); + if (!val || val->getBitWidth() != 32) + return emitNodeWarning(); + + return BoolAttr::get(ctx, val->getValue().getLimitedValue(1)); +} + +FailureOr LoopMetadataConversion::lookupIntNode(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return IntegerAttr(nullptr); + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold an i32 value"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + + llvm::ConstantInt *val = + llvm::mdconst::dyn_extract(property->getOperand(1)); + if (!val || val->getBitWidth() != 32) + return emitNodeWarning(); + + return IntegerAttr::get(IntegerType::get(ctx, 32), + val->getValue().getLimitedValue()); +} + +FailureOr LoopMetadataConversion::lookupMDNode(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + if (!property) + return nullptr; + + auto emitNodeWarning = [&]() { + return emitWarning(loc) + << "expected metadata node " << name << " to hold an MDNode"; + }; + + if (property->getNumOperands() != 2) + return emitNodeWarning(); + + auto *node = dyn_cast(property->getOperand(1)); + if (!node) + return emitNodeWarning(); + + return node; +} + +FailureOr> +LoopMetadataConversion::lookupMDNodes(StringRef name) { + const llvm::MDNode *property = lookupAndEraseProperty(name); + SmallVector res; + if (!property) + return res; + + auto emitNodeWarning = [&]() { + return emitWarning(loc) << "expected metadata node " << name + << " to hold one or multiple MDNodes"; + }; + + if (property->getNumOperands() < 2) + return emitNodeWarning(); + + for (unsigned i = 1, e = property->getNumOperands(); i < e; ++i) { + auto *node = dyn_cast(property->getOperand(i)); + if (!node) + return emitNodeWarning(); + res.push_back(node); + } + + return res; +} + +FailureOr +LoopMetadataConversion::lookupFollowupNode(StringRef name) { + auto node = lookupMDNode(name); + if (failed(node)) + return failure(); + if (*node == nullptr) + return LoopAnnotationAttr(nullptr); + + return loopAnnotationImporter.translateLoopAnnotation(*node, loc); +} + +static bool isEmptyOrNull(const Attribute attr) { return !attr; } + +template +static bool isEmptyOrNull(const SmallVectorImpl &vec) { + return vec.empty(); +} + +/// Helper function that only creates and attribute of type T if all argument +/// conversion were successfull and at least one of them holds a non-null value. +template +static T createIfNonNull(MLIRContext *ctx, const P &...args) { + bool anyFailed = (failed(args) || ...); + if (anyFailed) + return {}; + + bool allEmpty = (isEmptyOrNull(*args) && ...); + if (allEmpty) + return {}; + + return T::get(ctx, *args...); +} + +FailureOr LoopMetadataConversion::convertVectorizeAttr() { + FailureOr enable = + lookupBoolNode("llvm.loop.vectorize.enable", true); + FailureOr predicateEnable = + lookupBoolNode("llvm.loop.vectorize.predicate.enable"); + FailureOr scalableEnable = + lookupBoolNode("llvm.loop.vectorize.scalable.enable"); + FailureOr width = lookupIntNode("llvm.loop.vectorize.width"); + FailureOr followupVec = + lookupFollowupNode("llvm.loop.vectorize.followup_vectorized"); + FailureOr followupEpi = + lookupFollowupNode("llvm.loop.vectorize.followup_epilogue"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.vectorize.followup_all"); + + return createIfNonNull(ctx, enable, predicateEnable, + scalableEnable, width, followupVec, + followupEpi, followupAll); +} + +FailureOr LoopMetadataConversion::convertInterleaveAttr() { + FailureOr count = lookupIntNode("llvm.loop.interleave.count"); + return createIfNonNull(ctx, count); +} + +FailureOr LoopMetadataConversion::convertUnrollAttr() { + FailureOr disable = lookupBooleanUnitNode( + "llvm.loop.unroll.enable", "llvm.loop.unroll.disable", /*negated=*/true); + FailureOr count = lookupIntNode("llvm.loop.unroll.count"); + FailureOr runtimeDisable = + lookupUnitNode("llvm.loop.unroll.runtime.disable"); + FailureOr full = lookupUnitNode("llvm.loop.unroll.full"); + FailureOr followupUnrolled = + lookupFollowupNode("llvm.loop.unroll.followup_unrolled"); + FailureOr followupRemainder = + lookupFollowupNode("llvm.loop.unroll.followup_remainder"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.unroll.followup_all"); + + return createIfNonNull(ctx, disable, count, runtimeDisable, + full, followupUnrolled, + followupRemainder, followupAll); +} + +FailureOr +LoopMetadataConversion::convertUnrollAndJamAttr() { + FailureOr disable = lookupBooleanUnitNode( + "llvm.loop.unroll_and_jam.enable", "llvm.loop.unroll_and_jam.disable", + /*negated=*/true); + FailureOr count = + lookupIntNode("llvm.loop.unroll_and_jam.count"); + FailureOr followupOuter = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_outer"); + FailureOr followupInner = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_inner"); + FailureOr followupRemainderOuter = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer"); + FailureOr followupRemainderInner = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.unroll_and_jam.followup_all"); + return createIfNonNull( + ctx, disable, count, followupOuter, followupInner, followupRemainderOuter, + followupRemainderInner, followupAll); +} + +FailureOr LoopMetadataConversion::convertLICMAttr() { + FailureOr disable = lookupUnitNode("llvm.licm.disable"); + FailureOr versioningDisable = + lookupUnitNode("llvm.loop.licm_versioning.disable"); + return createIfNonNull(ctx, disable, versioningDisable); +} + +FailureOr LoopMetadataConversion::convertDistributeAttr() { + FailureOr disable = + lookupBoolNode("llvm.loop.distribute.enable", true); + FailureOr followupCoincident = + lookupFollowupNode("llvm.loop.distribute.followup_coincident"); + FailureOr followupSequential = + lookupFollowupNode("llvm.loop.distribute.followup_sequential"); + FailureOr followupFallback = + lookupFollowupNode("llvm.loop.distribute.followup_fallback"); + FailureOr followupAll = + lookupFollowupNode("llvm.loop.distribute.followup_all"); + return createIfNonNull(ctx, disable, followupCoincident, + followupSequential, + followupFallback, followupAll); +} + +FailureOr LoopMetadataConversion::convertPipelineAttr() { + FailureOr disable = lookupBoolNode("llvm.loop.pipeline.disable"); + FailureOr initiationinterval = + lookupIntNode("llvm.loop.pipeline.initiationinterval"); + return createIfNonNull(ctx, disable, initiationinterval); +} + +FailureOr LoopMetadataConversion::convertPeeledAttr() { + FailureOr count = lookupIntNode("llvm.loop.peeled.count"); + return createIfNonNull(ctx, count); +} + +FailureOr LoopMetadataConversion::convertUnswitchAttr() { + FailureOr partialDisable = + lookupUnitNode("llvm.loop.unswitch.partial.disable"); + return createIfNonNull(ctx, partialDisable); +} + +FailureOr> +LoopMetadataConversion::convertParallelAccesses() { + FailureOr> nodes = + lookupMDNodes("llvm.loop.parallel_accesses"); + if (failed(nodes)) + return failure(); + SmallVector refs; + for (llvm::MDNode *node : *nodes) { + FailureOr> accessGroups = + loopAnnotationImporter.lookupAccessGroupAttrs(node); + if (failed(accessGroups)) + return emitWarning(loc) << "could not lookup access group"; + llvm::append_range(refs, *accessGroups); + } + return refs; +} + +LoopAnnotationAttr LoopMetadataConversion::convert() { + if (failed(initPropertyMap())) + return {}; + + FailureOr disableNonForced = + lookupUnitNode("llvm.loop.disable_nonforced"); + FailureOr vecAttr = convertVectorizeAttr(); + FailureOr interleaveAttr = convertInterleaveAttr(); + FailureOr unrollAttr = convertUnrollAttr(); + FailureOr unrollAndJamAttr = convertUnrollAndJamAttr(); + FailureOr licmAttr = convertLICMAttr(); + FailureOr distributeAttr = convertDistributeAttr(); + FailureOr pipelineAttr = convertPipelineAttr(); + FailureOr peeledAttr = convertPeeledAttr(); + FailureOr unswitchAttr = convertUnswitchAttr(); + FailureOr mustProgress = lookupUnitNode("llvm.loop.mustprogress"); + FailureOr isVectorized = + lookupIntNodeAsBoolAttr("llvm.loop.isvectorized"); + FailureOr> parallelAccesses = + convertParallelAccesses(); + + // Drop the metadata if there are parts that cannot be imported. + if (!propertyMap.empty()) { + for (auto name : propertyMap.keys()) + emitWarning(loc) << "unknown loop annotation " << name; + return {}; + } + + return createIfNonNull( + ctx, disableNonForced, vecAttr, interleaveAttr, unrollAttr, + unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, peeledAttr, + unswitchAttr, mustProgress, isVectorized, parallelAccesses); +} + +LoopAnnotationAttr +LoopAnnotationImporter::translateLoopAnnotation(const llvm::MDNode *node, + Location loc) { + if (!node) + return {}; + + // Note: This check is necessary to distinguish between failed translations + // and not yet attempted translations. + auto it = loopMetadataMapping.find(node); + if (it != loopMetadataMapping.end()) + return it->getSecond(); + + LoopAnnotationAttr attr = LoopMetadataConversion(node, loc, *this).convert(); + + mapLoopMetadata(node, attr); + return attr; +} + +LogicalResult LoopAnnotationImporter::translateAccessGroup( + const llvm::MDNode *node, Location loc, MetadataOp metadataOp) { + SmallVector accessGroups; + if (!node->getNumOperands()) + accessGroups.push_back(node); + for (const llvm::MDOperand &operand : node->operands()) { + auto *childNode = dyn_cast(operand); + if (!childNode) + return failure(); + accessGroups.push_back(cast(operand.get())); + } + + // Convert all entries of the access group list to access group operations. + for (const llvm::MDNode *accessGroup : accessGroups) { + if (accessGroupMapping.count(accessGroup)) + continue; + // Verify the access group node is distinct and empty. + if (accessGroup->getNumOperands() != 0 || !accessGroup->isDistinct()) + return emitWarning(loc) + << "expected an access group node to be empty and distinct"; + + OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToEnd(&metadataOp.getBody().back()); + auto groupOp = builder.create( + loc, llvm::formatv("group_{0}", accessGroupMapping.size()).str()); + // Add a mapping from the access group node to the symbol reference pointing + // to the newly created operation. + accessGroupMapping[accessGroup] = SymbolRefAttr::get( + builder.getContext(), metadataOp.getSymName(), + FlatSymbolRefAttr::get(builder.getContext(), groupOp.getSymName())); + } + return success(); +} + +FailureOr> +LoopAnnotationImporter::lookupAccessGroupAttrs(const llvm::MDNode *node) const { + // An access group node is either a single access group or an access group + // list. + SmallVector accessGroups; + if (!node->getNumOperands()) + accessGroups.push_back(accessGroupMapping.lookup(node)); + for (const llvm::MDOperand &operand : node->operands()) { + auto *node = cast(operand.get()); + accessGroups.push_back(accessGroupMapping.lookup(node)); + } + // Exit if one of the access group node lookups failed. + if (llvm::is_contained(accessGroups, nullptr)) + return failure(); + return accessGroups; +} diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h new file mode 100644 index 000000000000..5d69a63a2150 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h @@ -0,0 +1,71 @@ +//===- LoopAnnotationImporter.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the translation between LLVMIR loop metadata and the +// corresponding MLIR representation. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_ +#define MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_ + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Target/LLVMIR/ModuleImport.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// A helper class that converts llvm.loop metadata nodes into corresponding +/// LoopAnnotationAttrs and llvm.access.group nodes into +/// AccessGroupMetadataOps. +class LoopAnnotationImporter { +public: + explicit LoopAnnotationImporter(OpBuilder &builder) : builder(builder) {} + LoopAnnotationAttr translateLoopAnnotation(const llvm::MDNode *node, + Location loc); + + /// Converts all LLVM access groups starting from node to MLIR access group + /// operations mested in the region of metadataOp. It stores a mapping from + /// every nested access group nod to the symbol pointing to the translated + /// operation. Returns success if all conversions succeed and failure + /// otherwise. + LogicalResult translateAccessGroup(const llvm::MDNode *node, Location loc, + MetadataOp metadataOp); + + /// Returns the symbol references pointing to the access group operations that + /// map to the access group nodes starting from the access group metadata + /// node. Returns failure, if any of the symbol references cannot be found. + FailureOr> + lookupAccessGroupAttrs(const llvm::MDNode *node) const; + +private: + /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. + LoopAnnotationAttr lookupLoopMetadata(const llvm::MDNode *node) const { + return loopMetadataMapping.lookup(node); + } + + void mapLoopMetadata(const llvm::MDNode *metadata, LoopAnnotationAttr attr) { + auto result = loopMetadataMapping.try_emplace(metadata, attr); + (void)result; + assert(result.second && + "attempting to map loop options that was already mapped"); + } + + OpBuilder &builder; + DenseMap loopMetadataMapping; + /// Mapping between original LLVM access group metadata nodes and the symbol + /// references pointing to the imported MLIR access group operations. + DenseMap accessGroupMapping; +}; + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#endif // MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_ diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp new file mode 100644 index 000000000000..4a535dcc3a8d --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -0,0 +1,292 @@ +//===- LoopAnnotationTranslation.cpp - Loop annotation export -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "LoopAnnotationTranslation.h" + +using namespace mlir; +using namespace mlir::LLVM; +using namespace mlir::LLVM::detail; + +namespace { +/// Helper class that keeps the state of one attribute to metadata conversion. +struct LoopAnnotationConversion { + LoopAnnotationConversion(LoopAnnotationAttr attr, Operation *op, + LoopAnnotationTranslation &loopAnnotationTranslation, + llvm::LLVMContext &ctx) + : attr(attr), op(op), + loopAnnotationTranslation(loopAnnotationTranslation), ctx(ctx) {} + + /// Converts this struct's loop annotation into a corresponding LLVMIR + /// metadata representation. + llvm::MDNode *convert(); + + /// Conversion functions for different payload attribute kinds. + void addUnitNode(StringRef name); + void addUnitNode(StringRef name, BoolAttr attr); + void addI32NodeWithVal(StringRef name, uint32_t val); + void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false); + void convertI32Node(StringRef name, IntegerAttr attr); + void convertFollowupNode(StringRef name, LoopAnnotationAttr attr); + + /// Conversion functions for each for each loop annotation sub-attribute. + void convertLoopOptions(LoopVectorizeAttr options); + void convertLoopOptions(LoopInterleaveAttr options); + void convertLoopOptions(LoopUnrollAttr options); + void convertLoopOptions(LoopUnrollAndJamAttr options); + void convertLoopOptions(LoopLICMAttr options); + void convertLoopOptions(LoopDistributeAttr options); + void convertLoopOptions(LoopPipelineAttr options); + void convertLoopOptions(LoopPeeledAttr options); + void convertLoopOptions(LoopUnswitchAttr options); + + LoopAnnotationAttr attr; + Operation *op; + LoopAnnotationTranslation &loopAnnotationTranslation; + llvm::LLVMContext &ctx; + llvm::SmallVector metadataNodes; +}; +} // namespace + +void LoopAnnotationConversion::addUnitNode(StringRef name) { + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name)})); +} + +void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) { + if (attr && attr.getValue()) + addUnitNode(name); +} + +void LoopAnnotationConversion::addI32NodeWithVal(StringRef name, uint32_t val) { + llvm::Constant *cstValue = llvm::ConstantInt::get( + llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false); + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + +void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr, + bool negated) { + if (!attr) + return; + bool val = negated ^ attr.getValue(); + llvm::Constant *cstValue = llvm::ConstantInt::getBool(ctx, val); + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), + llvm::ConstantAsMetadata::get(cstValue)})); +} + +void LoopAnnotationConversion::convertI32Node(StringRef name, + IntegerAttr attr) { + if (!attr) + return; + addI32NodeWithVal(name, attr.getInt()); +} + +void LoopAnnotationConversion::convertFollowupNode(StringRef name, + LoopAnnotationAttr attr) { + if (!attr) + return; + + llvm::MDNode *node = + loopAnnotationTranslation.translateLoopAnnotation(attr, op); + + metadataNodes.push_back( + llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), node})); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) { + convertBoolNode("llvm.loop.vectorize.enable", options.getDisable(), true); + convertBoolNode("llvm.loop.vectorize.predicate.enable", + options.getPredicateEnable()); + convertBoolNode("llvm.loop.vectorize.scalable.enable", + options.getScalableEnable()); + convertI32Node("llvm.loop.vectorize.width", options.getWidth()); + convertFollowupNode("llvm.loop.vectorize.followup_vectorized", + options.getFollowupVectorized()); + convertFollowupNode("llvm.loop.vectorize.followup_epilogue", + options.getFollowupEpilogue()); + convertFollowupNode("llvm.loop.vectorize.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopInterleaveAttr options) { + convertI32Node("llvm.loop.interleave.count", options.getCount()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) { + if (auto disable = options.getDisable()) + addUnitNode(disable.getValue() ? "llvm.loop.unroll.disable" + : "llvm.loop.unroll.enable"); + convertI32Node("llvm.loop.unroll.count", options.getCount()); + convertBoolNode("llvm.loop.unroll.runtime.disable", + options.getRuntimeDisable()); + addUnitNode("llvm.loop.unroll.full", options.getFull()); + convertFollowupNode("llvm.loop.unroll.followup_unrolled", + options.getFollowupUnrolled()); + convertFollowupNode("llvm.loop.unroll.followup_remainder", + options.getFollowupRemainder()); + convertFollowupNode("llvm.loop.unroll.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions( + LoopUnrollAndJamAttr options) { + if (auto disable = options.getDisable()) + addUnitNode(disable.getValue() ? "llvm.loop.unroll_and_jam.disable" + : "llvm.loop.unroll_and_jam.enable"); + convertI32Node("llvm.loop.unroll_and_jam.count", options.getCount()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_outer", + options.getFollowupOuter()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_inner", + options.getFollowupInner()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer", + options.getFollowupRemainderOuter()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner", + options.getFollowupRemainderInner()); + convertFollowupNode("llvm.loop.unroll_and_jam.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopLICMAttr options) { + addUnitNode("llvm.licm.disable", options.getDisable()); + addUnitNode("llvm.loop.licm_versioning.disable", + options.getVersioningDisable()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopDistributeAttr options) { + convertBoolNode("llvm.loop.distribute.enable", options.getDisable(), true); + convertFollowupNode("llvm.loop.distribute.followup_coincident", + options.getFollowupCoincident()); + convertFollowupNode("llvm.loop.distribute.followup_sequential", + options.getFollowupSequential()); + convertFollowupNode("llvm.loop.distribute.followup_fallback", + options.getFollowupFallback()); + convertFollowupNode("llvm.loop.distribute.followup_all", + options.getFollowupAll()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) { + convertBoolNode("llvm.loop.pipeline.disable", options.getDisable()); + convertI32Node("llvm.loop.pipeline.initiationinterval", + options.getInitiationinterval()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopPeeledAttr options) { + convertI32Node("llvm.loop.peeled.count", options.getCount()); +} + +void LoopAnnotationConversion::convertLoopOptions(LoopUnswitchAttr options) { + addUnitNode("llvm.loop.unswitch.partial.disable", + options.getPartialDisable()); +} + +llvm::MDNode *LoopAnnotationConversion::convert() { + + // Reserve operand 0 for loop id self reference. + auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt); + metadataNodes.push_back(dummy.get()); + + addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced()); + addUnitNode("llvm.loop.mustprogress", attr.getMustProgress()); + // "isvectorized" is encoded as an i32 value. + if (BoolAttr isVectorized = attr.getIsVectorized()) + addI32NodeWithVal("llvm.loop.isvectorized", isVectorized.getValue()); + + if (auto options = attr.getVectorize()) + convertLoopOptions(options); + if (auto options = attr.getInterleave()) + convertLoopOptions(options); + if (auto options = attr.getUnroll()) + convertLoopOptions(options); + if (auto options = attr.getUnrollAndJam()) + convertLoopOptions(options); + if (auto options = attr.getLicm()) + convertLoopOptions(options); + if (auto options = attr.getDistribute()) + convertLoopOptions(options); + if (auto options = attr.getPipeline()) + convertLoopOptions(options); + if (auto options = attr.getPeeled()) + convertLoopOptions(options); + if (auto options = attr.getUnswitch()) + convertLoopOptions(options); + + ArrayRef parallelAccessGroups = attr.getParallelAccesses(); + if (!parallelAccessGroups.empty()) { + SmallVector parallelAccess; + parallelAccess.push_back( + llvm::MDString::get(ctx, "llvm.loop.parallel_accesses")); + for (SymbolRefAttr accessGroupRef : parallelAccessGroups) + parallelAccess.push_back( + loopAnnotationTranslation.getAccessGroup(op, accessGroupRef)); + metadataNodes.push_back(llvm::MDNode::get(ctx, parallelAccess)); + } + + // Create loop options and set the first operand to itself. + llvm::MDNode *loopMD = llvm::MDNode::get(ctx, metadataNodes); + loopMD->replaceOperandWith(0, loopMD); + + return loopMD; +} + +llvm::MDNode * +LoopAnnotationTranslation::translateLoopAnnotation(LoopAnnotationAttr attr, + Operation *op) { + if (!attr) + return nullptr; + + llvm::MDNode *loopMD = lookupLoopMetadata(attr); + if (loopMD) + return loopMD; + + loopMD = + LoopAnnotationConversion(attr, op, *this, this->llvmModule.getContext()) + .convert(); + // Store a map from this Attribute to the LLVM metadata in case we + // encounter it again. + mapLoopMetadata(attr, loopMD); + return loopMD; +} + +LogicalResult LoopAnnotationTranslation::createAccessGroupMetadata() { + mlirModule->walk([&](LLVM::MetadataOp metadatas) { + metadatas.walk([&](LLVM::AccessGroupMetadataOp op) { + llvm::MDNode *accessGroup = + llvm::MDNode::getDistinct(llvmModule.getContext(), {}); + accessGroupMetadataMapping.insert({op, accessGroup}); + }); + }); + return success(); +} + +llvm::MDNode * +LoopAnnotationTranslation::getAccessGroup(Operation *op, + SymbolRefAttr accessGroupRef) const { + auto metadataName = accessGroupRef.getRootReference(); + auto accessGroupName = accessGroupRef.getLeafReference(); + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + auto *accessGroupOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); + return accessGroupMetadataMapping.lookup(accessGroupOp); +} + +llvm::MDNode * +LoopAnnotationTranslation::getAccessGroups(Operation *op, + ArrayAttr accessGroupRefs) const { + if (!accessGroupRefs || accessGroupRefs.empty()) + return nullptr; + + SmallVector groupMDs; + for (SymbolRefAttr groupRef : accessGroupRefs.getAsRange()) + groupMDs.push_back(getAccessGroup(op, groupRef)); + if (groupMDs.size() == 1) + return llvm::cast(groupMDs.front()); + return llvm::MDNode::get(llvmModule.getContext(), groupMDs); +} diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h new file mode 100644 index 000000000000..4de54f399830 --- /dev/null +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h @@ -0,0 +1,77 @@ +//===- LoopAnnotationTranslation.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the translation between an MLIR loop annotations and +// the corresponding LLVMIR metadata representation. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ +#define MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Target/LLVMIR/ModuleTranslation.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// A helper class that converts LoopAnnotationAttrs and AccessGroupMetadataOps +/// into a corresponding llvm::MDNodes. +class LoopAnnotationTranslation { +public: + LoopAnnotationTranslation(Operation *mlirModule, llvm::Module &llvmModule) + : mlirModule(mlirModule), llvmModule(llvmModule) {} + + llvm::MDNode *translateLoopAnnotation(LoopAnnotationAttr attr, Operation *op); + + /// Traverses the global access group metadata operation in the `mlirModule` + /// and creates corresponding LLVM metadata nodes. + LogicalResult createAccessGroupMetadata(); + + /// Returns the LLVM metadata corresponding to a symbol reference to an mlir + /// LLVM dialect access group operation. + llvm::MDNode *getAccessGroup(Operation *op, + SymbolRefAttr accessGroupRef) const; + + /// Returns the LLVM metadata corresponding to a list of symbol reference to + /// an mlir LLVM dialect access group operation. Returns nullptr if + /// `accessGroupRefs` is null or empty. + llvm::MDNode *getAccessGroups(Operation *op, ArrayAttr accessGroupRefs) const; + +private: + /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute. + llvm::MDNode *lookupLoopMetadata(Attribute options) const { + return loopMetadataMapping.lookup(options); + } + + void mapLoopMetadata(Attribute options, llvm::MDNode *metadata) { + auto result = loopMetadataMapping.try_emplace(options, metadata); + (void)result; + assert(result.second && + "attempting to map loop options that was already mapped"); + } + + /// Mapping from an attribute describing loop metadata to its LLVM metadata. + /// The metadata is attached to Latch block branches with this attribute. + DenseMap loopMetadataMapping; + + /// Mapping from an access group metadata operation to its LLVM metadata. + /// This map is populated on module entry and is used to annotate loops (as + /// identified via their branches) and contained memory accesses. + DenseMap accessGroupMetadataMapping; + + Operation *mlirModule; + llvm::Module &llvmModule; +}; + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#endif // MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_ diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 17cdd5bb8855..fd4b69e63bab 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -14,7 +14,9 @@ #include "mlir/Target/LLVMIR/ModuleImport.h" #include "mlir/Target/LLVMIR/Import.h" +#include "AttrKindDetail.h" #include "DebugImporter.h" +#include "LoopAnnotationImporter.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" @@ -71,6 +73,12 @@ static constexpr StringRef getGlobalDtorsVarName() { return "llvm.global_dtors"; } +/// Returns the symbol name for the module-level metadata operation. It must not +/// conflict with the user namespace. +static constexpr StringRef getGlobalMetadataOpName() { + return "__llvm_global_metadata"; +} + /// Returns a supported MLIR floating point type of the given bit width or null /// if the bit width is not supported. static FloatType getDLFloatType(MLIRContext &ctx, int32_t bitwidth) { @@ -90,136 +98,27 @@ static FloatType getDLFloatType(MLIRContext &ctx, int32_t bitwidth) { } } -static ICmpPredicate getICmpPredicate(llvm::CmpInst::Predicate pred) { - switch (pred) { - default: - llvm_unreachable("incorrect comparison predicate"); - case llvm::CmpInst::Predicate::ICMP_EQ: - return LLVM::ICmpPredicate::eq; - case llvm::CmpInst::Predicate::ICMP_NE: - return LLVM::ICmpPredicate::ne; - case llvm::CmpInst::Predicate::ICMP_SLT: - return LLVM::ICmpPredicate::slt; - case llvm::CmpInst::Predicate::ICMP_SLE: - return LLVM::ICmpPredicate::sle; - case llvm::CmpInst::Predicate::ICMP_SGT: - return LLVM::ICmpPredicate::sgt; - case llvm::CmpInst::Predicate::ICMP_SGE: - return LLVM::ICmpPredicate::sge; - case llvm::CmpInst::Predicate::ICMP_ULT: - return LLVM::ICmpPredicate::ult; - case llvm::CmpInst::Predicate::ICMP_ULE: - return LLVM::ICmpPredicate::ule; - case llvm::CmpInst::Predicate::ICMP_UGT: - return LLVM::ICmpPredicate::ugt; - case llvm::CmpInst::Predicate::ICMP_UGE: - return LLVM::ICmpPredicate::uge; - } - llvm_unreachable("incorrect integer comparison predicate"); -} - -static FCmpPredicate getFCmpPredicate(llvm::CmpInst::Predicate pred) { - switch (pred) { - default: - llvm_unreachable("incorrect comparison predicate"); - case llvm::CmpInst::Predicate::FCMP_FALSE: - return LLVM::FCmpPredicate::_false; - case llvm::CmpInst::Predicate::FCMP_TRUE: - return LLVM::FCmpPredicate::_true; - case llvm::CmpInst::Predicate::FCMP_OEQ: - return LLVM::FCmpPredicate::oeq; - case llvm::CmpInst::Predicate::FCMP_ONE: - return LLVM::FCmpPredicate::one; - case llvm::CmpInst::Predicate::FCMP_OLT: - return LLVM::FCmpPredicate::olt; - case llvm::CmpInst::Predicate::FCMP_OLE: - return LLVM::FCmpPredicate::ole; - case llvm::CmpInst::Predicate::FCMP_OGT: - return LLVM::FCmpPredicate::ogt; - case llvm::CmpInst::Predicate::FCMP_OGE: - return LLVM::FCmpPredicate::oge; - case llvm::CmpInst::Predicate::FCMP_ORD: - return LLVM::FCmpPredicate::ord; - case llvm::CmpInst::Predicate::FCMP_ULT: - return LLVM::FCmpPredicate::ult; - case llvm::CmpInst::Predicate::FCMP_ULE: - return LLVM::FCmpPredicate::ule; - case llvm::CmpInst::Predicate::FCMP_UGT: - return LLVM::FCmpPredicate::ugt; - case llvm::CmpInst::Predicate::FCMP_UGE: - return LLVM::FCmpPredicate::uge; - case llvm::CmpInst::Predicate::FCMP_UNO: - return LLVM::FCmpPredicate::uno; - case llvm::CmpInst::Predicate::FCMP_UEQ: - return LLVM::FCmpPredicate::ueq; - case llvm::CmpInst::Predicate::FCMP_UNE: - return LLVM::FCmpPredicate::une; - } - llvm_unreachable("incorrect floating point comparison predicate"); -} - -static AtomicOrdering getLLVMAtomicOrdering(llvm::AtomicOrdering ordering) { - switch (ordering) { - case llvm::AtomicOrdering::NotAtomic: - return LLVM::AtomicOrdering::not_atomic; - case llvm::AtomicOrdering::Unordered: - return LLVM::AtomicOrdering::unordered; - case llvm::AtomicOrdering::Monotonic: - return LLVM::AtomicOrdering::monotonic; - case llvm::AtomicOrdering::Acquire: - return LLVM::AtomicOrdering::acquire; - case llvm::AtomicOrdering::Release: - return LLVM::AtomicOrdering::release; - case llvm::AtomicOrdering::AcquireRelease: - return LLVM::AtomicOrdering::acq_rel; - case llvm::AtomicOrdering::SequentiallyConsistent: - return LLVM::AtomicOrdering::seq_cst; - } - llvm_unreachable("incorrect atomic ordering"); -} - -static AtomicBinOp getLLVMAtomicBinOp(llvm::AtomicRMWInst::BinOp binOp) { - switch (binOp) { - case llvm::AtomicRMWInst::Xchg: - return LLVM::AtomicBinOp::xchg; - case llvm::AtomicRMWInst::Add: - return LLVM::AtomicBinOp::add; - case llvm::AtomicRMWInst::Sub: - return LLVM::AtomicBinOp::sub; - case llvm::AtomicRMWInst::And: - return LLVM::AtomicBinOp::_and; - case llvm::AtomicRMWInst::Nand: - return LLVM::AtomicBinOp::nand; - case llvm::AtomicRMWInst::Or: - return LLVM::AtomicBinOp::_or; - case llvm::AtomicRMWInst::Xor: - return LLVM::AtomicBinOp::_xor; - case llvm::AtomicRMWInst::Max: - return LLVM::AtomicBinOp::max; - case llvm::AtomicRMWInst::Min: - return LLVM::AtomicBinOp::min; - case llvm::AtomicRMWInst::UMax: - return LLVM::AtomicBinOp::umax; - case llvm::AtomicRMWInst::UMin: - return LLVM::AtomicBinOp::umin; - case llvm::AtomicRMWInst::FAdd: - return LLVM::AtomicBinOp::fadd; - case llvm::AtomicRMWInst::FSub: - return LLVM::AtomicBinOp::fsub; - default: - llvm_unreachable("unsupported atomic binary operation"); - } -} - -/// Converts the sync scope identifier of `fenceInst` to the string -/// representation necessary to build the LLVM dialect fence operation. -static StringRef getLLVMSyncScope(llvm::FenceInst *fenceInst) { - llvm::LLVMContext &llvmContext = fenceInst->getContext(); - SmallVector syncScopeNames; - llvmContext.getSyncScopeNames(syncScopeNames); - for (StringRef name : syncScopeNames) - if (fenceInst->getSyncScopeID() == llvmContext.getOrInsertSyncScopeID(name)) - return name; +/// Converts the sync scope identifier of `inst` to the string representation +/// necessary to build an atomic LLVM dialect operation. Returns the empty +/// string if the operation has either no sync scope or the default system-level +/// sync scope attached. The atomic operations only set their sync scope +/// attribute if they have a non-default sync scope attached. +static StringRef getLLVMSyncScope(llvm::Instruction *inst) { + std::optional syncScopeID = + llvm::getAtomicSyncScopeID(inst); + if (!syncScopeID) + return ""; + + // Search the sync scope name for the given identifier. The default + // system-level sync scope thereby maps to the empty string. + SmallVector syncScopeName; + llvm::LLVMContext &llvmContext = inst->getContext(); + llvmContext.getSyncScopeNames(syncScopeName); + auto *it = llvm::find_if(syncScopeName, [&](StringRef name) { + return *syncScopeID == llvmContext.getOrInsertSyncScopeID(name); + }); + if (it != syncScopeName.end()) + return *it; llvm_unreachable("incorrect sync scope identifier"); } @@ -355,27 +254,20 @@ ModuleImport::ModuleImport(ModuleOp mlirModule, mlirModule(mlirModule), llvmModule(std::move(llvmModule)), iface(mlirModule->getContext()), typeTranslator(*mlirModule->getContext()), - debugImporter(std::make_unique(mlirModule)) { + debugImporter(std::make_unique(mlirModule)), + loopAnnotationImporter( + std::make_unique(builder)) { builder.setInsertionPointToStart(mlirModule.getBody()); } -MetadataOp ModuleImport::getTBAAMetadataOp() { - if (tbaaMetadataOp) - return tbaaMetadataOp; +MetadataOp ModuleImport::getGlobalMetadataOp() { + if (globalMetadataOp) + return globalMetadataOp; OpBuilder::InsertionGuard guard(builder); - Location loc = mlirModule.getLoc(); - builder.setInsertionPointToEnd(mlirModule.getBody()); - tbaaMetadataOp = builder.create(loc, getTBAAMetadataOpName()); - - return tbaaMetadataOp; -} - -std::string ModuleImport::getNewTBAANodeName(StringRef basename) { - return (Twine("tbaa_") + Twine(basename) + Twine('_') + - Twine(tbaaNodeCounter++)) - .str(); + return globalMetadataOp = builder.create( + mlirModule.getLoc(), getGlobalMetadataOpName()); } LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { @@ -534,10 +426,18 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { return true; }; + // Helper to compute a unique symbol name that includes the given `baseName`. + // Uses the size of the mapping to unique the symbol name. + auto getUniqueSymbolName = [&](StringRef baseName) { + return (Twine("tbaa_") + Twine(baseName) + Twine('_') + + Twine(tbaaMapping.size())) + .str(); + }; + // Insert new operations at the end of the MetadataOp. OpBuilder::InsertionGuard guard(builder); - builder.setInsertionPointToEnd(&getTBAAMetadataOp().getBody().back()); - StringAttr metadataOpName = SymbolTable::getSymbolName(getTBAAMetadataOp()); + builder.setInsertionPointToEnd(&getGlobalMetadataOp().getBody().back()); + StringAttr metadataOpName = SymbolTable::getSymbolName(getGlobalMetadataOp()); // On the first walk, create SymbolRefAttr's and map them // to nodes in `nodesToConvert`. @@ -550,7 +450,7 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { // The root nodes do not have operands, so we can create // the TBAARootMetadataOp on the first walk. auto rootNode = builder.create( - loc, getNewTBAANodeName("root"), identity.value()); + loc, getUniqueSymbolName("root"), identity.value()); tbaaMapping.try_emplace(current, FlatSymbolRefAttr::get(rootNode)); continue; } @@ -559,7 +459,7 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { return failure(); tbaaMapping.try_emplace( current, FlatSymbolRefAttr::get(builder.getContext(), - getNewTBAANodeName("type_desc"))); + getUniqueSymbolName("type_desc"))); continue; } if (std::optional isValid = isTagNode(current)) { @@ -571,7 +471,7 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { current, SymbolRefAttr::get( builder.getContext(), metadataOpName, FlatSymbolRefAttr::get(builder.getContext(), - getNewTBAANodeName("tag")))); + getUniqueSymbolName("tag")))); continue; } return emitError(loc) << "unsupported TBAA node format: " @@ -611,21 +511,135 @@ LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) { return success(); } +LogicalResult +ModuleImport::processAccessGroupMetadata(const llvm::MDNode *node) { + Location loc = mlirModule.getLoc(); + if (failed(loopAnnotationImporter->translateAccessGroup( + node, loc, getGlobalMetadataOp()))) + return emitError(loc) << "unsupported access group node: " + << diagMD(node, llvmModule.get()); + return success(); +} + +LogicalResult +ModuleImport::processAliasScopeMetadata(const llvm::MDNode *node) { + Location loc = mlirModule.getLoc(); + // Helper that verifies the node has a self reference operand. + auto verifySelfRef = [](const llvm::MDNode *node) { + return node->getNumOperands() != 0 && + node == dyn_cast(node->getOperand(0)); + }; + // Helper that verifies the given operand is a string or does not exist. + auto verifyDescription = [](const llvm::MDNode *node, unsigned idx) { + return idx >= node->getNumOperands() || + isa(node->getOperand(idx)); + }; + // Helper that creates an alias scope domain operation. + auto createAliasScopeDomainOp = [&](const llvm::MDNode *aliasDomain) { + StringAttr description = nullptr; + if (aliasDomain->getNumOperands() >= 2) + if (auto *operand = dyn_cast(aliasDomain->getOperand(1))) + description = builder.getStringAttr(operand->getString()); + std::string name = llvm::formatv("domain_{0}", aliasScopeMapping.size()); + return builder.create(loc, name, description); + }; + + // Collect the alias scopes and domains to translate them. + for (const llvm::MDOperand &operand : node->operands()) { + if (const auto *scope = dyn_cast(operand)) { + llvm::AliasScopeNode aliasScope(scope); + const llvm::MDNode *domain = aliasScope.getDomain(); + + // Verify the scope node points to valid scope metadata which includes + // verifying its domain. Perform the verification before looking it up in + // the alias scope mapping since it could have been inserted as a domain + // node before. + if (!verifySelfRef(scope) || !domain || !verifyDescription(scope, 2)) + return emitError(loc) << "unsupported alias scope node: " + << diagMD(scope, llvmModule.get()); + if (!verifySelfRef(domain) || !verifyDescription(domain, 1)) + return emitError(loc) << "unsupported alias domain node: " + << diagMD(domain, llvmModule.get()); + + if (aliasScopeMapping.count(scope)) + continue; + + // Set the insertion point to the end of the global metadata operation. + MetadataOp metadataOp = getGlobalMetadataOp(); + StringAttr metadataOpName = + SymbolTable::getSymbolName(getGlobalMetadataOp()); + OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToEnd(&metadataOp.getBody().back()); + + // Convert the domain metadata node if it has not been translated before. + auto it = aliasScopeMapping.find(aliasScope.getDomain()); + if (it == aliasScopeMapping.end()) { + auto aliasScopeDomainOp = createAliasScopeDomainOp(domain); + auto symbolRef = SymbolRefAttr::get( + builder.getContext(), metadataOpName, + FlatSymbolRefAttr::get(builder.getContext(), + aliasScopeDomainOp.getSymName())); + it = aliasScopeMapping.try_emplace(domain, symbolRef).first; + } + + // Convert the scope metadata node if it has not been converted before. + StringAttr description = nullptr; + if (!aliasScope.getName().empty()) + description = builder.getStringAttr(aliasScope.getName()); + std::string name = llvm::formatv("scope_{0}", aliasScopeMapping.size()); + auto aliasScopeOp = builder.create( + loc, name, it->getSecond().getLeafReference().getValue(), + description); + auto symbolRef = + SymbolRefAttr::get(builder.getContext(), metadataOpName, + FlatSymbolRefAttr::get(builder.getContext(), + aliasScopeOp.getSymName())); + aliasScopeMapping.try_emplace(aliasScope.getNode(), symbolRef); + } + } + return success(); +} + +FailureOr> +ModuleImport::lookupAliasScopeAttrs(const llvm::MDNode *node) const { + SmallVector aliasScopes; + aliasScopes.reserve(node->getNumOperands()); + for (const llvm::MDOperand &operand : node->operands()) { + auto *node = cast(operand.get()); + aliasScopes.push_back(aliasScopeMapping.lookup(node)); + } + // Return failure if one of the alias scope lookups failed. + if (llvm::is_contained(aliasScopes, nullptr)) + return failure(); + return aliasScopes; +} + LogicalResult ModuleImport::convertMetadata() { OpBuilder::InsertionGuard guard(builder); builder.setInsertionPointToEnd(mlirModule.getBody()); - for (const llvm::Function &func : llvmModule->functions()) + for (const llvm::Function &func : llvmModule->functions()) { for (const llvm::Instruction &inst : llvm::instructions(func)) { - llvm::AAMDNodes nodes = inst.getAAMetadata(); - if (!nodes) - continue; + // Convert access group metadata nodes. + if (llvm::MDNode *node = + inst.getMetadata(llvm::LLVMContext::MD_access_group)) + if (failed(processAccessGroupMetadata(node))) + return failure(); - if (const llvm::MDNode *tbaaMD = nodes.TBAA) - if (failed(processTBAAMetadata(tbaaMD))) + // Convert alias analysis metadata nodes. + llvm::AAMDNodes aliasAnalysisNodes = inst.getAAMetadata(); + if (!aliasAnalysisNodes) + continue; + if (aliasAnalysisNodes.TBAA) + if (failed(processTBAAMetadata(aliasAnalysisNodes.TBAA))) + return failure(); + if (aliasAnalysisNodes.Scope) + if (failed(processAliasScopeMetadata(aliasAnalysisNodes.Scope))) + return failure(); + if (aliasAnalysisNodes.NoAlias) + if (failed(processAliasScopeMetadata(aliasAnalysisNodes.NoAlias))) return failure(); - // TODO: only TBAA metadata is currently supported. } - + } return success(); } @@ -933,37 +947,57 @@ ModuleImport::convertGlobalCtorsAndDtors(llvm::GlobalVariable *globalVar) { SetVector ModuleImport::getConstantsToConvert(llvm::Constant *constant) { - // Traverse the constant dependencies in post order. - SmallVector workList; - SmallVector orderedList; - workList.push_back(constant); + // Return the empty set if the constant has been translated before. + if (valueMapping.count(constant)) + return {}; + + // Traverse the constants in post-order and stop the traversal if a constant + // already has a `valueMapping` from an earlier constant translation or if the + // constant is traversed a second time. + SetVector orderedSet; + SetVector workList; + DenseMap> adjacencyLists; + workList.insert(constant); while (!workList.empty()) { - llvm::Constant *current = workList.pop_back_val(); - // Skip constants that have been converted before and store all other ones. - if (valueMapping.count(current)) + llvm::Constant *current = workList.back(); + // Collect all dependencies of the current constant and add them to the + // adjacency list if none has been computed before. + auto adjacencyIt = adjacencyLists.find(current); + if (adjacencyIt == adjacencyLists.end()) { + adjacencyIt = adjacencyLists.try_emplace(current).first; + // Add all constant operands to the adjacency list and skip any other + // values such as basic block addresses. + for (llvm::Value *operand : current->operands()) + if (auto *constDependency = dyn_cast(operand)) + adjacencyIt->getSecond().push_back(constDependency); + // Use the getElementValue method to add the dependencies of zero + // initialized aggregate constants since they do not take any operands. + if (auto *constAgg = dyn_cast(current)) { + unsigned numElements = constAgg->getElementCount().getFixedValue(); + for (unsigned i = 0, e = numElements; i != e; ++i) + adjacencyIt->getSecond().push_back(constAgg->getElementValue(i)); + } + } + // Add the current constant to the `orderedSet` of the traversed nodes if + // all its dependencies have been traversed before. Additionally, remove the + // constant from the `workList` and continue the traversal. + if (adjacencyIt->getSecond().empty()) { + orderedSet.insert(current); + workList.pop_back(); continue; - orderedList.push_back(current); - // Add the current constant's dependencies to the work list. Only add - // constant dependencies and skip any other values such as basic block - // addresses. - for (llvm::Value *operand : current->operands()) - if (auto *constDependency = dyn_cast(operand)) - workList.push_back(constDependency); - // Use the `getElementValue` method to add the dependencies of zero - // initialized aggregate constants since they do not take any operands. - if (auto *constAgg = dyn_cast(current)) { - unsigned numElements = constAgg->getElementCount().getFixedValue(); - for (unsigned i = 0, e = numElements; i != e; ++i) - workList.push_back(constAgg->getElementValue(i)); } + // Add the next dependency from the adjacency list to the `workList` and + // continue the traversal. Remove the dependency from the adjacency list to + // mark that it has been processed. Only enqueue the dependency if it has no + // `valueMapping` from an earlier translation and if it has not been + // enqueued before. + llvm::Constant *dependency = adjacencyIt->getSecond().pop_back_val(); + if (valueMapping.count(dependency) || workList.count(dependency) || + orderedSet.count(dependency)) + continue; + workList.insert(dependency); } - // Add the constants in reverse post order to the result set to ensure all - // dependencies are satisfied. Avoid storing duplicates since LLVM constants - // are uniqued and only one `valueMapping` entry per constant is possible. - SetVector orderedSet; - for (llvm::Constant *orderedConst : llvm::reverse(orderedList)) - orderedSet.insert(orderedConst); return orderedSet; } @@ -1093,11 +1127,8 @@ FailureOr ModuleImport::convertConstantExpr(llvm::Constant *constant) { } FailureOr ModuleImport::convertValue(llvm::Value *value) { - // A value may be wrapped as metadata, for example, when passed to a debug - // intrinsic. Unwrap these values before the conversion. - if (auto *nodeAsVal = dyn_cast(value)) - if (auto *node = dyn_cast(nodeAsVal->getMetadata())) - value = node->getValue(); + assert(!isa(value) && + "expected value to not be metadata"); // Return the mapped value if it has been converted before. if (valueMapping.count(value)) @@ -1113,6 +1144,27 @@ FailureOr ModuleImport::convertValue(llvm::Value *value) { return emitError(loc) << "unhandled value: " << diag(*value); } +FailureOr ModuleImport::convertMetadataValue(llvm::Value *value) { + // A value may be wrapped as metadata, for example, when passed to a debug + // intrinsic. Unwrap these values before the conversion. + auto *nodeAsVal = dyn_cast(value); + if (!nodeAsVal) + return failure(); + auto *node = dyn_cast(nodeAsVal->getMetadata()); + if (!node) + return failure(); + value = node->getValue(); + + // Return the mapped value if it has been converted before. + if (valueMapping.count(value)) + return lookupValue(value); + + // Convert constants such as immediate values that have no mapping yet. + if (auto *constant = dyn_cast(value)) + return convertConstantExpr(constant); + return failure(); +} + FailureOr> ModuleImport::convertValues(ArrayRef values) { SmallVector remapped; @@ -1482,6 +1534,45 @@ void ModuleImport::processFunctionAttributes(llvm::Function *func, processPassthroughAttrs(func, funcOp); } +DictionaryAttr +ModuleImport::convertParameterAttribute(llvm::AttributeSet llvmParamAttrs, + OpBuilder &builder) { + SmallVector paramAttrs; + for (auto [llvmKind, mlirName] : getAttrKindToNameMapping()) { + auto llvmAttr = llvmParamAttrs.getAttribute(llvmKind); + // Skip attributes that are not attached. + if (!llvmAttr.isValid()) + continue; + Attribute mlirAttr; + if (llvmAttr.isTypeAttribute()) + mlirAttr = TypeAttr::get(convertType(llvmAttr.getValueAsType())); + else if (llvmAttr.isIntAttribute()) + mlirAttr = builder.getI64IntegerAttr(llvmAttr.getValueAsInt()); + else if (llvmAttr.isEnumAttribute()) + mlirAttr = builder.getUnitAttr(); + else + llvm_unreachable("unexpected parameter attribute kind"); + paramAttrs.push_back(builder.getNamedAttr(mlirName, mlirAttr)); + } + + return builder.getDictionaryAttr(paramAttrs); +} + +void ModuleImport::convertParameterAttributes(llvm::Function *func, + LLVMFuncOp funcOp, + OpBuilder &builder) { + auto llvmAttrs = func->getAttributes(); + for (size_t i = 0, e = funcOp.getNumArguments(); i < e; ++i) { + llvm::AttributeSet llvmArgAttrs = llvmAttrs.getParamAttrs(i); + funcOp.setArgAttrs(i, convertParameterAttribute(llvmArgAttrs, builder)); + } + // Convert the result attributes and attach them wrapped in an ArrayAttribute + // to the funcOp. + llvm::AttributeSet llvmResAttr = llvmAttrs.getRetAttrs(); + funcOp.setResAttrsAttr( + builder.getArrayAttr(convertParameterAttribute(llvmResAttr, builder))); +} + LogicalResult ModuleImport::processFunction(llvm::Function *func) { clearBlockAndValueMapping(); @@ -1505,35 +1596,7 @@ LogicalResult ModuleImport::processFunction(llvm::Function *func) { // Set the function debug information if available. debugImporter->translate(func, funcOp); - for (const auto &it : llvm::enumerate(functionType.getParams())) { - llvm::SmallVector argAttrs; - if (auto *type = func->getParamByValType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back( - NamedAttribute(builder.getStringAttr(LLVMDialect::getByValAttrName()), - TypeAttr::get(mlirType))); - } - if (auto *type = func->getParamByRefType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back( - NamedAttribute(builder.getStringAttr(LLVMDialect::getByRefAttrName()), - TypeAttr::get(mlirType))); - } - if (auto *type = func->getParamStructRetType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back(NamedAttribute( - builder.getStringAttr(LLVMDialect::getStructRetAttrName()), - TypeAttr::get(mlirType))); - } - if (auto *type = func->getParamInAllocaType(it.index())) { - Type mlirType = convertType(type); - argAttrs.push_back(NamedAttribute( - builder.getStringAttr(LLVMDialect::getInAllocaAttrName()), - TypeAttr::get(mlirType))); - } - - funcOp.setArgAttrs(it.index(), argAttrs); - } + convertParameterAttributes(func, funcOp, builder); if (FlatSymbolRefAttr personality = getPersonalityAsAttr(func)) funcOp.setPersonalityAttr(personality); @@ -1609,6 +1672,17 @@ LogicalResult ModuleImport::processBasicBlock(llvm::BasicBlock *bb, return success(); } +FailureOr> +ModuleImport::lookupAccessGroupAttrs(const llvm::MDNode *node) const { + return loopAnnotationImporter->lookupAccessGroupAttrs(node); +} + +LoopAnnotationAttr +ModuleImport::translateLoopAnnotationAttr(const llvm::MDNode *node, + Location loc) const { + return loopAnnotationImporter->translateLoopAnnotation(node, loc); +} + OwningOpRef mlir::translateLLVMIRToModule(std::unique_ptr llvmModule, MLIRContext *context) { diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index ecb9908c6cb6..04eddde310cf 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -13,7 +13,9 @@ #include "mlir/Target/LLVMIR/ModuleTranslation.h" +#include "AttrKindDetail.h" #include "DebugTranslation.h" +#include "LoopAnnotationTranslation.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h" @@ -25,10 +27,10 @@ #include "mlir/Support/LLVM.h" #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h" #include "mlir/Target/LLVMIR/TypeToLLVM.h" -#include "llvm/ADT/TypeSwitch.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" +#include "llvm/ADT/TypeSwitch.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" @@ -214,6 +216,16 @@ convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr, if (type.getNumElements() == 0) return nullptr; + // Check that the raw data size matches what is expected for the scalar size. + // TODO: in theory, we could repack the data here to keep constructing from + // raw data. + // TODO: we may also need to consider endianness when cross-compiling to an + // architecture where it is different. + unsigned elementByteSize = denseElementsAttr.getRawData().size() / + denseElementsAttr.getNumElements(); + if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) + return nullptr; + // Compute the shape of all dimensions but the innermost. Note that the // innermost dimension may be that of the vector element type. bool hasVectorElementType = type.getElementType().isa(); @@ -409,6 +421,8 @@ ModuleTranslation::ModuleTranslation(Operation *module, : mlirModule(module), llvmModule(std::move(llvmModule)), debugTranslation( std::make_unique(module, *this->llvmModule)), + loopAnnotationTranslation(std::make_unique( + module, *this->llvmModule)), typeTranslator(this->llvmModule->getContext()), iface(module->getContext()) { assert(satisfiesLLVMModule(mlirModule) && @@ -435,7 +449,6 @@ void ModuleTranslation::forgetMapping(Region ®ion) { branchMapping.erase(&op); if (isa(op)) globalsMapping.erase(&op); - accessGroupMetadataMapping.erase(&op); llvm::append_range( toProcess, llvm::map_range(op.getRegions(), [](Region &r) { return &r; })); @@ -896,6 +909,33 @@ static void convertFunctionAttributes(LLVMFuncOp func, llvmFunc->setMemoryEffects(newMemEffects); } +llvm::AttrBuilder +ModuleTranslation::convertParameterAttrs(DictionaryAttr paramAttrs) { + llvm::AttrBuilder attrBuilder(llvmModule->getContext()); + + for (auto [llvmKind, mlirName] : getAttrKindToNameMapping()) { + Attribute attr = paramAttrs.get(mlirName); + // Skip attributes that are not present. + if (!attr) + continue; + + // NOTE: C++17 does not support capturing structured bindings. + llvm::Attribute::AttrKind llvmKindCap = llvmKind; + + llvm::TypeSwitch(attr) + .Case([&](auto typeAttr) { + attrBuilder.addTypeAttr(llvmKindCap, + convertType(typeAttr.getValue())); + }) + .Case([&](auto intAttr) { + attrBuilder.addRawIntAttr(llvmKindCap, intAttr.getInt()); + }) + .Case([&](auto) { attrBuilder.addAttribute(llvmKindCap); }); + } + + return attrBuilder; +} + LogicalResult ModuleTranslation::convertFunctionSignatures() { // Declare all functions first because there may be function calls that form a // call graph with cycles, or global initializers that reference functions. @@ -918,149 +958,16 @@ LogicalResult ModuleTranslation::convertFunctionSignatures() { // Convert result attributes. if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) { - llvm::AttrBuilder retAttrs(llvmFunc->getContext()); DictionaryAttr resultAttrs = allResultAttrs[0].cast(); - for (const NamedAttribute &attr : resultAttrs) { - StringAttr name = attr.getName(); - if (name == LLVMDialect::getAlignAttrName()) { - auto alignAmount = attr.getValue().cast(); - retAttrs.addAlignmentAttr(llvm::Align(alignAmount.getInt())); - } else if (name == LLVMDialect::getNoAliasAttrName()) { - retAttrs.addAttribute(llvm::Attribute::NoAlias); - } else if (name == LLVMDialect::getNoUndefAttrName()) { - retAttrs.addAttribute(llvm::Attribute::NoUndef); - } else if (name == LLVMDialect::getSExtAttrName()) { - retAttrs.addAttribute(llvm::Attribute::SExt); - } else if (name == LLVMDialect::getZExtAttrName()) { - retAttrs.addAttribute(llvm::Attribute::ZExt); - } - } - llvmFunc->addRetAttrs(retAttrs); + llvmFunc->addRetAttrs(convertParameterAttrs(resultAttrs)); } // Convert argument attributes. - unsigned int argIdx = 0; - for (auto [mlirArgTy, llvmArg] : - llvm::zip(function.getArgumentTypes(), llvmFunc->args())) { - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getNoAliasAttrName())) { - // NB: Attribute already verified to be boolean, so check if we can - // indeed attach the attribute to this argument, based on its type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.noalias attribute attached to LLVM non-pointer argument"); - llvmArg.addAttr(llvm::Attribute::AttrKind::NoAlias); - } - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getReadonlyAttrName())) { - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.readonly attribute attached to LLVM non-pointer argument"); - llvmArg.addAttr(llvm::Attribute::AttrKind::ReadOnly); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getAlignAttrName())) { - // NB: Attribute already verified to be int, so check if we can indeed - // attach the attribute to this argument, based on its type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.align attribute attached to LLVM non-pointer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAlignmentAttr(llvm::Align(attr.getInt()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getStructRetAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.sret attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.sret attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addStructRetAttr(convertType(attr.getValue()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getByValAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.byval attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.byval attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addByValAttr(convertType(attr.getValue()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getByRefAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.byref attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.byref attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addByRefAttr(convertType(attr.getValue()))); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getInAllocaAttrName())) { - auto argTy = mlirArgTy.dyn_cast(); - if (!argTy) - return function.emitError( - "llvm.inalloca attribute attached to LLVM non-pointer argument"); - if (!argTy.isOpaque() && argTy.getElementType() != attr.getValue()) - return function.emitError( - "llvm.inalloca attribute attached to LLVM pointer " - "argument of a different type"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addInAllocaAttr(convertType(attr.getValue()))); - } - - if (auto attr = - function.getArgAttrOfType(argIdx, "llvm.nest")) { - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.nest attribute attached to LLVM non-pointer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::Nest)); - } - - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getNoUndefAttrName())) { - // llvm.noundef can be added to any argument type. - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::NoUndef)); - } - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getSExtAttrName())) { - // llvm.signext can be added to any integer argument type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.signext attribute attached to LLVM non-integer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::SExt)); - } - if (auto attr = function.getArgAttrOfType( - argIdx, LLVMDialect::getZExtAttrName())) { - // llvm.zeroext can be added to any integer argument type. - if (!mlirArgTy.isa()) - return function.emitError( - "llvm.zeroext attribute attached to LLVM non-integer argument"); - llvmArg.addAttrs(llvm::AttrBuilder(llvmArg.getContext()) - .addAttribute(llvm::Attribute::ZExt)); + for (auto [argIdx, llvmArg] : llvm::enumerate(llvmFunc->args())) { + if (DictionaryAttr argAttrs = function.getArgAttrDict(argIdx)) { + llvm::AttrBuilder attrBuilder = convertParameterAttrs(argAttrs); + llvmArg.addAttrs(attrBuilder); } - - ++argIdx; } // Forward the pass-through attributes to LLVM. @@ -1086,48 +993,21 @@ LogicalResult ModuleTranslation::convertFunctions() { return success(); } -llvm::MDNode * -ModuleTranslation::getAccessGroup(Operation &opInst, - SymbolRefAttr accessGroupRef) const { - auto metadataName = accessGroupRef.getRootReference(); - auto accessGroupName = accessGroupRef.getLeafReference(); - auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - opInst.getParentOp(), metadataName); - auto *accessGroupOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName); - return accessGroupMetadataMapping.lookup(accessGroupOp); -} - LogicalResult ModuleTranslation::createAccessGroupMetadata() { - mlirModule->walk([&](LLVM::MetadataOp metadatas) { - metadatas.walk([&](LLVM::AccessGroupMetadataOp op) { - llvm::LLVMContext &ctx = llvmModule->getContext(); - llvm::MDNode *accessGroup = llvm::MDNode::getDistinct(ctx, {}); - accessGroupMetadataMapping.insert({op, accessGroup}); - }); - }); - return success(); + return loopAnnotationTranslation->createAccessGroupMetadata(); } void ModuleTranslation::setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst) { - auto accessGroups = + auto populateGroupsMetadata = [&](ArrayAttr groupRefs) { + if (llvm::MDNode *node = + loopAnnotationTranslation->getAccessGroups(op, groupRefs)) + inst->setMetadata(llvm::LLVMContext::MD_access_group, node); + }; + + auto groupRefs = op->getAttrOfType(LLVMDialect::getAccessGroupsAttrName()); - if (accessGroups && !accessGroups.empty()) { - llvm::Module *module = inst->getModule(); - SmallVector metadatas; - for (SymbolRefAttr accessGroupRef : - accessGroups.getAsRange()) - metadatas.push_back(getAccessGroup(*op, accessGroupRef)); - - llvm::MDNode *unionMD = nullptr; - if (metadatas.size() == 1) - unionMD = llvm::cast(metadatas.front()); - else if (metadatas.size() >= 2) - unionMD = llvm::MDNode::get(module->getContext(), metadatas); - - inst->setMetadata(module->getMDKindID("llvm.access.group"), unionMD); - } + populateGroupsMetadata(groupRefs); } LogicalResult ModuleTranslation::createAliasScopeMetadata() { @@ -1168,12 +1048,12 @@ LogicalResult ModuleTranslation::createAliasScopeMetadata() { } llvm::MDNode * -ModuleTranslation::getAliasScope(Operation &opInst, +ModuleTranslation::getAliasScope(Operation *op, SymbolRefAttr aliasScopeRef) const { StringAttr metadataName = aliasScopeRef.getRootReference(); StringAttr scopeName = aliasScopeRef.getLeafReference(); auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - opInst.getParentOp(), metadataName); + op->getParentOp(), metadataName); Operation *aliasScopeOp = SymbolTable::lookupNearestSymbolFrom(metadataOp, scopeName); return aliasScopeMetadataMapping.lookup(aliasScopeOp); @@ -1181,50 +1061,61 @@ ModuleTranslation::getAliasScope(Operation &opInst, void ModuleTranslation::setAliasScopeMetadata(Operation *op, llvm::Instruction *inst) { - auto populateScopeMetadata = [this, op, inst](StringRef attrName, - StringRef llvmMetadataName) { - auto scopes = op->getAttrOfType(attrName); - if (!scopes || scopes.empty()) + auto populateScopeMetadata = [&](ArrayAttr scopeRefs, unsigned kind) { + if (!scopeRefs || scopeRefs.empty()) return; llvm::Module *module = inst->getModule(); SmallVector scopeMDs; - for (SymbolRefAttr scopeRef : scopes.getAsRange()) - scopeMDs.push_back(getAliasScope(*op, scopeRef)); - llvm::MDNode *unionMD = llvm::MDNode::get(module->getContext(), scopeMDs); - inst->setMetadata(module->getMDKindID(llvmMetadataName), unionMD); + for (SymbolRefAttr scopeRef : scopeRefs.getAsRange()) + scopeMDs.push_back(getAliasScope(op, scopeRef)); + llvm::MDNode *node = llvm::MDNode::get(module->getContext(), scopeMDs); + inst->setMetadata(kind, node); }; - populateScopeMetadata(LLVMDialect::getAliasScopesAttrName(), "alias.scope"); - populateScopeMetadata(LLVMDialect::getNoAliasScopesAttrName(), "noalias"); + auto aliasScopeRefs = + op->getAttrOfType(LLVMDialect::getAliasScopesAttrName()); + populateScopeMetadata(aliasScopeRefs, llvm::LLVMContext::MD_alias_scope); + + auto noaliasScopeRefs = + op->getAttrOfType(LLVMDialect::getNoAliasScopesAttrName()); + populateScopeMetadata(noaliasScopeRefs, llvm::LLVMContext::MD_noalias); } -llvm::MDNode *ModuleTranslation::getTBAANode(Operation &memOp, +llvm::MDNode *ModuleTranslation::getTBAANode(Operation *op, SymbolRefAttr tagRef) const { StringAttr metadataName = tagRef.getRootReference(); StringAttr tagName = tagRef.getLeafReference(); auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - memOp.getParentOp(), metadataName); + op->getParentOp(), metadataName); Operation *tagOp = SymbolTable::lookupNearestSymbolFrom(metadataOp, tagName); return tbaaMetadataMapping.lookup(tagOp); } void ModuleTranslation::setTBAAMetadata(Operation *op, llvm::Instruction *inst) { - auto tbaa = op->getAttrOfType(LLVMDialect::getTBAAAttrName()); - if (!tbaa || tbaa.empty()) - return; - // LLVM IR currently does not support attaching more than one - // TBAA access tag to a memory accessing instruction. - // It may be useful to support this in future, but for the time being - // just ignore the metadata if MLIR operation has multiple access tags. - if (tbaa.size() > 1) { - op->emitWarning() << "TBAA access tags were not translated, because LLVM " - "IR only supports a single tag per instruction"; - return; - } - SymbolRefAttr tagRef = tbaa[0].cast(); - llvm::MDNode *tagNode = getTBAANode(*op, tagRef); - inst->setMetadata(llvm::LLVMContext::MD_tbaa, tagNode); + auto populateTBAAMetadata = [&](std::optional tagRefs) { + if (!tagRefs || tagRefs->empty()) + return; + + // LLVM IR currently does not support attaching more than one + // TBAA access tag to a memory accessing instruction. + // It may be useful to support this in future, but for the time being + // just ignore the metadata if MLIR operation has multiple access tags. + if (tagRefs->size() > 1) { + op->emitWarning() << "TBAA access tags were not translated, because LLVM " + "IR only supports a single tag per instruction"; + return; + } + + SymbolRefAttr tagRef = (*tagRefs)[0].cast(); + llvm::MDNode *node = getTBAANode(op, tagRef); + inst->setMetadata(llvm::LLVMContext::MD_tbaa, node); + }; + + llvm::TypeSwitch(op) + .Case( + [&](auto memOp) { populateTBAAMetadata(memOp.getTbaa()); }) + .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); } LogicalResult ModuleTranslation::createTBAAMetadata() { @@ -1319,6 +1210,19 @@ LogicalResult ModuleTranslation::createTBAAMetadata() { return success(); } +void ModuleTranslation::setLoopMetadata(Operation *op, + llvm::Instruction *inst) { + LoopAnnotationAttr attr = + TypeSwitch(op) + .Case( + [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); }); + if (!attr) + return; + llvm::MDNode *loopMD = + loopAnnotationTranslation->translateLoopAnnotation(attr, op); + inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD); +} + llvm::Type *ModuleTranslation::convertType(Type type) { return typeTranslator.translateType(type); } diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp index 186118b2e067..85e7c87b4c7b 100644 --- a/mlir/lib/Transforms/ViewOpGraph.cpp +++ b/mlir/lib/Transforms/ViewOpGraph.cpp @@ -9,6 +9,7 @@ #include "mlir/Transforms/ViewOpGraph.h" #include "mlir/IR/Block.h" +#include "mlir/IR/BuiltinOps.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Operation.h" #include "mlir/Pass/Pass.h" @@ -16,6 +17,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/Support/Format.h" #include "llvm/Support/GraphWriter.h" +#include "llvm/Support/raw_ostream.h" #include #include @@ -28,6 +30,7 @@ using namespace mlir; static const StringRef kLineStyleControlFlow = "dashed"; static const StringRef kLineStyleDataFlow = "solid"; +static const StringRef kLineStyleRegionControlFlow = "bold"; static const StringRef kShapeNode = "ellipse"; static const StringRef kShapeNone = "plain"; @@ -82,6 +85,10 @@ struct Node { /// Note: See https://www.graphviz.org/doc/info/lang.html for more information /// about the Graphviz DOT language. class PrintOpPass : public impl::ViewOpGraphBase { +private: + llvm::DenseMap blockFirstNodeMap; + llvm::DenseMap blockLastNodeMap; + public: PrintOpPass(raw_ostream &os) : os(os) {} PrintOpPass(const PrintOpPass &o) : PrintOpPass(o.os.getOStream()) {} @@ -250,20 +257,61 @@ class PrintOpPass : public impl::ViewOpGraphBase { /// Process a block. Emit a cluster and one node per block argument and /// operation inside the cluster. void processBlock(Block &block) { + + // If we are processing the `Block` containing the functions, we should not + // skip all the intermediate operations (the `LLVMFuncOp` themself). We can + // check this by disabling the feature if the parent `Operation` of the + // current block is the `Module` itself. + bool IsModuleBlock = llvm::isa(block.getParentOp()); + + // Prepare the name for the block node + std::string nodeName; + llvm::raw_string_ostream blockNameStr(nodeName); + block.printAsOperand(blockNameStr); emitClusterStmt([&]() { for (BlockArgument &blockArg : block.getArguments()) valueToNode[blockArg] = emitNodeStmt(getLabel(blockArg)); // Emit a node for each operation. std::optional prevNode; - for (Operation &op : block) { - Node nextNode = processOperation(&op); - if (printControlFlowEdges && prevNode) - emitEdgeStmt(*prevNode, nextNode, /*label=*/"", + + // Separately handle the `onlyEntryAndExitOperations` option, by only + // printing the first and last operation in a `mlir::Block`. + if (!IsModuleBlock and onlyEntryAndExitOperations + and block.getOperations().size() > 2) { + Operation &firstOp = block.front(); + Node firstNode = processOperation(&firstOp); + blockFirstNodeMap[&block] = firstNode; + + Operation &lastOp = block.back(); + Node lastNode = processOperation(&lastOp); + blockLastNodeMap[&block] = lastNode; + + if (printControlFlowEdges) { + emitEdgeStmt(firstNode, lastNode, /*label=*/"", kLineStyleControlFlow); - prevNode = nextNode; + } + } else { + + // In all the other cases, we loop over all the operations. + for (Operation &op : block) { + Node nextNode = processOperation(&op); + if (printControlFlowEdges && prevNode) + emitEdgeStmt(*prevNode, nextNode, /*label=*/"", + kLineStyleControlFlow); + + // If we are at first iteration, save the first operand node for the + // incoming edges. + if (not prevNode) { + blockFirstNodeMap[&block] = nextNode; + } + prevNode = nextNode; + } + + // Save the last operation for the outgoing edges. + blockLastNodeMap[&block] = *prevNode; } - }); + }, nodeName); } /// Process an operation. If the operation has regions, emit a cluster. @@ -301,6 +349,22 @@ class PrintOpPass : public impl::ViewOpGraphBase { void processRegion(Region ®ion) { for (Block &block : region.getBlocks()) processBlock(block); + + // Print control flow edges between blocks if the option is activated. + if (printRegionControlFlowEdges) { + for (Block &block : region.getBlocks()) { + unsigned numSuccessors = block.getNumSuccessors(); + for (unsigned i = 0; i < numSuccessors; i++) { + Block *successor = block.getSuccessor(i); + assert(blockLastNodeMap.count(&block) == 1); + Node blockNode = blockLastNodeMap[&block]; + assert(blockFirstNodeMap.count(successor) == 1); + Node successorNode = blockFirstNodeMap[successor]; + emitEdgeStmt(blockNode, successorNode, + /*label=*/numSuccessors == 1 ? "" : std::to_string(i), kLineStyleRegionControlFlow); + } + } + } } /// Truncate long strings. diff --git a/mlir/test/CAPI/execution_engine.c b/mlir/test/CAPI/execution_engine.c index 96ad8eb1a402..582b2f8fe222 100644 --- a/mlir/test/CAPI/execution_engine.c +++ b/mlir/test/CAPI/execution_engine.c @@ -34,7 +34,7 @@ void lowerModuleToLLVM(MlirContext ctx, MlirModule module) { MlirPassManager pm = mlirPassManagerCreate(ctx); MlirOpPassManager opm = mlirPassManagerGetNestedUnder( pm, mlirStringRefCreateFromCString("func.func")); - mlirPassManagerAddOwnedPass(pm, mlirCreateConversionConvertFuncToLLVM()); + mlirPassManagerAddOwnedPass(pm, mlirCreateConversionConvertFuncToLLVMPass()); mlirOpPassManagerAddOwnedPass( opm, mlirCreateConversionArithToLLVMConversionPass()); MlirLogicalResult status = mlirPassManagerRun(pm, module); diff --git a/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir b/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir index 24d85e8596a4..3324dff9d997 100644 --- a/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir +++ b/mlir/test/Conversion/FuncToLLVM/calling-convention.mlir @@ -1,5 +1,5 @@ -// RUN: mlir-opt -convert-memref-to-llvm -llvm-request-c-wrappers -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s -// RUN: mlir-opt -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s --check-prefix=EMIT_C_ATTRIBUTE +// RUN: mlir-opt -finalize-memref-to-llvm -llvm-request-c-wrappers -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts %s | FileCheck %s --check-prefix=EMIT_C_ATTRIBUTE // This tests the default memref calling convention and the emission of C // wrappers. We don't need to separate runs because the wrapper-emission diff --git a/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir b/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir index 9db15b89ccc9..716ef210093d 100644 --- a/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir +++ b/mlir/test/Conversion/FuncToLLVM/convert-funcs.mlir @@ -52,7 +52,7 @@ func.func private @body(i32) func.func @indirect_const_call(%arg0: i32) { // CHECK-NEXT: %[[ADDR:.*]] = llvm.mlir.addressof @body : !llvm.ptr> %0 = constant @body : (i32) -> () -// CHECK-NEXT: llvm.call %[[ADDR]](%[[ARG0:.*]]) : (i32) -> () +// CHECK-NEXT: llvm.call %[[ADDR]](%[[ARG0:.*]]) : !llvm.ptr>, (i32) -> () call_indirect %0(%arg0) : (i32) -> () // CHECK-NEXT: llvm.return return @@ -60,7 +60,7 @@ func.func @indirect_const_call(%arg0: i32) { // CHECK-LABEL: llvm.func @indirect_call(%arg0: !llvm.ptr>, %arg1: f32) -> i32 { func.func @indirect_call(%arg0: (f32) -> i32, %arg1: f32) -> i32 { -// CHECK-NEXT: %0 = llvm.call %arg0(%arg1) : (f32) -> i32 +// CHECK-NEXT: %0 = llvm.call %arg0(%arg1) : !llvm.ptr>, (f32) -> i32 %0 = call_indirect %arg0(%arg1) : (f32) -> i32 // CHECK-NEXT: llvm.return %0 : i32 return %0 : i32 diff --git a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir index 8b9360a09900..826ca9540ae5 100644 --- a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir +++ b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-callers.mlir @@ -2,7 +2,7 @@ // CHECK: llvm.func @res_attrs_with_memref_return() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_memref_return -// CHECK: %{{.*}}: !llvm.ptr{{.*}} {test.returnOne} +// CHECK-NOT: test.returnOne func.func @res_attrs_with_memref_return() -> (memref {test.returnOne}) { %0 = memref.alloc() : memref return %0 : memref @@ -10,24 +10,30 @@ func.func @res_attrs_with_memref_return() -> (memref {test.returnOne}) { // CHECK: llvm.func @res_attrs_with_value_return() -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_value_return -// CHECK: -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func @res_attrs_with_value_return() -> (f32 {test.returnOne = 1}) { %0 = arith.constant 1.00 : f32 return %0 : f32 } -// CHECK: llvm.func @multiple_return() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func @multiple_return() -> (memref {test.returnOne = 1}, f32 {test.returnTwo = 2, test.returnThree = 3}) { %0 = memref.alloc() : memref %1 = arith.constant 1.00 : f32 return %0, %1 : memref, f32 } -// CHECK: llvm.func @multiple_return_missing_res_attr() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return_missing_res_attr() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return_missing_res_attr -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func @multiple_return_missing_res_attr() -> (memref {test.returnOne = 1}, i64, f32 {test.returnTwo = 2, test.returnThree = 3}) { %0 = memref.alloc() : memref %1 = arith.constant 2 : i64 @@ -37,7 +43,9 @@ func.func @multiple_return_missing_res_attr() -> (memref {test.returnOne = // CHECK: llvm.func @one_arg_attr_no_res_attrs_with_memref_return({{.*}}) -> !llvm.struct{{.*}} // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_no_res_attrs_with_memref_return -// CHECK: %{{.*}}: !llvm.ptr<{{.*}}>, %{{.*}}: !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref {test.argOne = 1}) -> memref { %0 = memref.alloc() : memref return %0 : memref @@ -45,7 +53,10 @@ func.func @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref {test // CHECK: llvm.func @one_arg_attr_one_res_attr_with_memref_return({{.*}}) -> (!llvm.struct<{{.*}}> {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_memref_return -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {test.returnOne = 1 : i64}, %{{.*}}: !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func @one_arg_attr_one_res_attr_with_memref_return(%arg0: memref {test.argOne = 1}) -> (memref {test.returnOne = 1}) { %0 = memref.alloc() : memref return %0 : memref @@ -53,15 +64,23 @@ func.func @one_arg_attr_one_res_attr_with_memref_return(%arg0: memref {test // CHECK: llvm.func @one_arg_attr_one_res_attr_with_value_return({{.*}}) -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_value_return -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {test.argOne = 1 : i64}) -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func @one_arg_attr_one_res_attr_with_value_return(%arg0: memref {test.argOne = 1}) -> (f32 {test.returnOne = 1}) { %0 = arith.constant 1.00 : f32 return %0 : f32 } -// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_arg_attr_multiple_res_attr -// CHECK: (%{{.*}}: !llvm.ptr<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}, %{{.*}}: !llvm.ptr<{{.*}}> {test.argZero = 0 : i64}, %{{.*}}: f32, %{{.*}}: i32 {test.argTwo = 2 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argZero = 0 : i64} +// CHECK-SAME: f32 +// CHECK-SAME: i32 {test.argTwo = 2 : i64}) func.func @multiple_arg_attr_multiple_res_attr(%arg0: memref {test.argZero = 0}, %arg1: f32, %arg2: i32 {test.argTwo = 2}) -> (f32, memref {test.returnOne = 1}, i32 {test.returnTwo = 2}) { %0 = arith.constant 1.00 : f32 %1 = memref.alloc() : memref @@ -71,8 +90,8 @@ func.func @multiple_arg_attr_multiple_res_attr(%arg0: memref {test.argZero // CHECK: llvm.func @drop_linkage_attr() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_drop_linkage_attr +// CHECK-SAME: !llvm.ptr // CHECK-NOT: llvm.linkage -// CHECK: %{{.*}}: !llvm.ptr{{.*}} {test.returnOne} func.func @drop_linkage_attr() -> (memref {test.returnOne}) attributes { llvm.linkage = #llvm.linkage } { %0 = memref.alloc() : memref return %0 : memref diff --git a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir index 9d73072aab9d..f43c23773d9d 100644 --- a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir +++ b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir @@ -2,46 +2,64 @@ // CHECK: llvm.func @res_attrs_with_memref_return() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_memref_return -// CHECK: !llvm.ptr{{.*}} {test.returnOne} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne func.func private @res_attrs_with_memref_return() -> (memref {test.returnOne}) // CHECK: llvm.func @res_attrs_with_value_return() -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_value_return -// CHECK: -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func private @res_attrs_with_value_return() -> (f32 {test.returnOne = 1}) -// CHECK: llvm.func @multiple_return() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return -// CHECK: (!llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func private @multiple_return() -> (memref {test.returnOne = 1}, f32 {test.returnTwo = 2, test.returnThree = 3}) -// CHECK: llvm.func @multiple_return_missing_res_attr() -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_return_missing_res_attr() -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return_missing_res_attr -// CHECK: (!llvm.ptr<{{.*}}> {llvm.struct_attrs = [{test.returnOne = 1 : i64}, {}, {test.returnThree = 3 : i64, test.returnTwo = 2 : i64}]}) +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-NOT: test.returnThree func.func private @multiple_return_missing_res_attr() -> (memref {test.returnOne = 1}, i64, f32 {test.returnTwo = 2, test.returnThree = 3}) // CHECK: llvm.func @one_arg_attr_no_res_attrs_with_memref_return({{.*}}) -> !llvm.struct{{.*}} // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_no_res_attrs_with_memref_return -// CHECK: !llvm.ptr<{{.*}}>, !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func private @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref {test.argOne = 1}) -> memref // CHECK: llvm.func @one_arg_attr_one_res_attr_with_memref_return({{.*}}) -> (!llvm.struct<{{.*}}> {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_memref_return -// CHECK: (!llvm.ptr<{{.*}}> {test.returnOne = 1 : i64}, !llvm.ptr<{{.*}}> {test.argOne = 1 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64}) func.func private @one_arg_attr_one_res_attr_with_memref_return(%arg0: memref {test.argOne = 1}) -> (memref {test.returnOne = 1}) // CHECK: llvm.func @one_arg_attr_one_res_attr_with_value_return({{.*}}) -> (f32 {test.returnOne = 1 : i64}) // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_value_return -// CHECK: (!llvm.ptr<{{.*}}> {test.argOne = 1 : i64}) -> (f32 {test.returnOne = 1 : i64}) +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argOne = 1 : i64} +// CHECK-SAME: -> (f32 {test.returnOne = 1 : i64}) func.func private @one_arg_attr_one_res_attr_with_value_return(%arg0: memref {test.argOne = 1}) -> (f32 {test.returnOne = 1}) -// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> (!llvm.struct<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}) +// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> !llvm.struct<{{.*}}> // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_arg_attr_multiple_res_attr -// CHECK: (!llvm.ptr<{{.*}}> {llvm.struct_attrs = [{}, {test.returnOne = 1 : i64}, {test.returnTwo = 2 : i64}]}, !llvm.ptr<{{.*}}> {test.argZero = 0 : i64}, f32, i32 {test.argTwo = 2 : i64} +// CHECK-SAME: !llvm.ptr +// CHECK-NOT: test.returnOne +// CHECK-NOT: test.returnTwo +// CHECK-SAME: !llvm.ptr +// CHECK-SAME: {test.argZero = 0 : i64} +// CHECK-SAME: f32 +// CHECK-SAME: i32 {test.argTwo = 2 : i64}) func.func private @multiple_arg_attr_multiple_res_attr(%arg0: memref {test.argZero = 0}, %arg1: f32, %arg2: i32 {test.argTwo = 2}) -> (f32, memref {test.returnOne = 1}, i32 {test.returnTwo = 2}) // CHECK: llvm.func weak @drop_linkage_attr() -> (!llvm.struct{{.*}} {test.returnOne}) // CHECK-LABEL: llvm.func @_mlir_ciface_drop_linkage_attr +// CHECK-SAME: !llvm.ptr // CHECK-NOT: llvm.linkage -// CHECK: !llvm.ptr{{.*}} {test.returnOne} func.func private @drop_linkage_attr() -> (memref {test.returnOne}) attributes { llvm.linkage = #llvm.linkage } diff --git a/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir index df10b317aa49..89c0268a0f72 100644 --- a/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir +++ b/mlir/test/Conversion/GPUCommon/lower-memcpy-to-gpu-runtime-calls.mlir @@ -7,8 +7,10 @@ module attributes {gpu.container_module} { // CHECK: %[[t0:.*]] = llvm.call @mgpuStreamCreate %t0 = gpu.wait async // CHECK: %[[size_bytes:.*]] = llvm.ptrtoint + // CHECK-NOT: llvm.addrspacecast // CHECK: %[[src:.*]] = llvm.bitcast - // CHECK: %[[dst:.*]] = llvm.bitcast + // CHECK: %[[addr_cast:.*]] = llvm.addrspacecast + // CHECK: %[[dst:.*]] = llvm.bitcast %[[addr_cast]] // CHECK: llvm.call @mgpuMemcpy(%[[dst]], %[[src]], %[[size_bytes]], %[[t0]]) %t1 = gpu.memcpy async [%t0] %dst, %src : memref<7xf32, 1>, memref<7xf32> // CHECK: llvm.call @mgpuStreamSynchronize(%[[t0]]) diff --git a/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir index ef5b6ef2c7bb..562c15583369 100644 --- a/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir +++ b/mlir/test/Conversion/GPUCommon/lower-memset-to-gpu-runtime-calls.mlir @@ -8,7 +8,8 @@ module attributes {gpu.container_module} { %t0 = gpu.wait async // CHECK: %[[size_bytes:.*]] = llvm.mlir.constant // CHECK: %[[value:.*]] = llvm.bitcast - // CHECK: %[[dst:.*]] = llvm.bitcast + // CHECK: %[[addr_cast:.*]] = llvm.addrspacecast + // CHECK: %[[dst:.*]] = llvm.bitcast %[[addr_cast]] // CHECK: llvm.call @mgpuMemset32(%[[dst]], %[[value]], %[[size_bytes]], %[[t0]]) %t1 = gpu.memset async [%t0] %dst, %value : memref<7xf32, 1>, f32 // CHECK: llvm.call @mgpuStreamSynchronize(%[[t0]]) diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir index f443f05795d2..5cc9b33d21f1 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-alloca-scope.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -convert-memref-to-llvm %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=1' %s | FileCheck %s // CHECK-LABEL: @empty func.func @empty() { diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir index 821f298ba4ce..6987bdcc104d 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-dynamic-memref-ops.mlir @@ -1,6 +1,6 @@ -// RUN: mlir-opt -split-input-file -convert-memref-to-llvm %s | FileCheck %s -// RUN: mlir-opt -split-input-file -convert-memref-to-llvm='use-aligned-alloc=1' %s | FileCheck %s --check-prefix=ALIGNED-ALLOC -// RUN: mlir-opt -split-input-file -convert-memref-to-llvm='index-bitwidth=32' %s | FileCheck --check-prefix=CHECK32 %s +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='use-opaque-pointers=1' %s | FileCheck %s +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='use-aligned-alloc=1 use-opaque-pointers=1' %s | FileCheck %s --check-prefix=ALIGNED-ALLOC +// RUN: mlir-opt -split-input-file -finalize-memref-to-llvm='index-bitwidth=32 use-opaque-pointers=1' %s | FileCheck --check-prefix=CHECK32 %s // CHECK-LABEL: func @mixed_alloc( // CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) @@ -11,22 +11,21 @@ func.func @mixed_alloc(%arg0: index, %arg1: index) -> memref { // CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-NEXT: %[[st0:.*]] = llvm.mul %[[N]], %[[c42]] : i64 // CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[st0]], %[[M]] : i64 -// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr -// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> %0 = memref.alloc(%arg0, %arg1) : memref return %0 : memref } @@ -35,9 +34,8 @@ func.func @mixed_alloc(%arg0: index, %arg1: index) -> memref { // CHECK-LABEL: func @mixed_dealloc func.func @mixed_dealloc(%arg0: memref) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return } @@ -51,20 +49,19 @@ func.func @dynamic_alloc(%arg0: index, %arg1: index) -> memref { // CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] // CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[N]], %[[M]] : i64 -// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr -// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.alloc(%arg0, %arg1) : memref return %0 : memref } @@ -78,28 +75,28 @@ func.func @dynamic_alloca(%arg0: index, %arg1: index) -> memref { // CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] // CHECK-NEXT: %[[st1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-NEXT: %[[num_elems:.*]] = llvm.mul %[[N]], %[[M]] : i64 -// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK-NEXT: %[[allocated:.*]] = llvm.alloca %[[sz_bytes]] x f32 : (i64) -> !llvm.ptr -// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: %[[allocated:.*]] = llvm.alloca %[[sz_bytes]] x f32 : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[allocated]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[st1]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.alloca(%arg0, %arg1) : memref // Test with explicitly specified alignment. llvm.alloca takes care of the // alignment. The same pointer is thus used for allocation and aligned // accesses. -// CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr -// CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr +// CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> memref.alloca(%arg0, %arg1) {alignment = 32} : memref return %0 : memref } @@ -108,9 +105,8 @@ func.func @dynamic_alloca(%arg0: index, %arg1: index) -> memref { // CHECK-LABEL: func @dynamic_dealloc func.func @dynamic_dealloc(%arg0: memref) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return } @@ -124,12 +120,11 @@ func.func @stdlib_aligned_alloc(%N : index) -> memref<32x18xf32> { // ALIGNED-ALLOC-NEXT: %[[sz2:.*]] = llvm.mlir.constant(18 : index) : i64 // ALIGNED-ALLOC-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // ALIGNED-ALLOC-NEXT: %[[num_elems:.*]] = llvm.mlir.constant(576 : index) : i64 -// ALIGNED-ALLOC-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// ALIGNED-ALLOC-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// ALIGNED-ALLOC-NEXT: %[[bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// ALIGNED-ALLOC-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// ALIGNED-ALLOC-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// ALIGNED-ALLOC-NEXT: %[[bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 // ALIGNED-ALLOC-NEXT: %[[alignment:.*]] = llvm.mlir.constant(32 : index) : i64 -// ALIGNED-ALLOC-NEXT: %[[allocated:.*]] = llvm.call @aligned_alloc(%[[alignment]], %[[bytes]]) : (i64, i64) -> !llvm.ptr -// ALIGNED-ALLOC-NEXT: llvm.bitcast %[[allocated]] : !llvm.ptr to !llvm.ptr +// ALIGNED-ALLOC-NEXT: %[[allocated:.*]] = llvm.call @aligned_alloc(%[[alignment]], %[[bytes]]) : (i64, i64) -> !llvm.ptr %0 = memref.alloc() {alignment = 32} : memref<32x18xf32> // Do another alloc just to test that we have a unique declaration for // aligned_alloc. @@ -170,12 +165,12 @@ func.func @stdlib_aligned_alloc(%N : index) -> memref<32x18xf32> { func.func @mixed_load(%mixed : memref<42x?xf32>, %i : index, %j : index) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr -> f32 %0 = memref.load %mixed[%i, %j] : memref<42x?xf32> return } @@ -187,12 +182,12 @@ func.func @mixed_load(%mixed : memref<42x?xf32>, %i : index, %j : index) { func.func @dynamic_load(%dynamic : memref, %i : index, %j : index) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.load %[[addr]] : !llvm.ptr -> f32 %0 = memref.load %dynamic[%i, %j] : memref return } @@ -204,25 +199,25 @@ func.func @dynamic_load(%dynamic : memref, %i : index, %j : index) { func.func @prefetch(%A : memref, %i : index, %j : index) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK-NEXT: [[C1:%.*]] = llvm.mlir.constant(1 : i32) : i32 // CHECK-NEXT: [[C3:%.*]] = llvm.mlir.constant(3 : i32) : i32 // CHECK-NEXT: [[C1_1:%.*]] = llvm.mlir.constant(1 : i32) : i32 -// CHECK-NEXT: "llvm.intr.prefetch"(%[[addr]], [[C1]], [[C3]], [[C1_1]]) : (!llvm.ptr, i32, i32, i32) -> () +// CHECK-NEXT: "llvm.intr.prefetch"(%[[addr]], [[C1]], [[C3]], [[C1_1]]) : (!llvm.ptr, i32, i32, i32) -> () memref.prefetch %A[%i, %j], write, locality<3>, data : memref // CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: [[C0_1:%.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: [[C1_2:%.*]] = llvm.mlir.constant(1 : i32) : i32 -// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0]], [[C0_1]], [[C1_2]]) : (!llvm.ptr, i32, i32, i32) -> () +// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0]], [[C0_1]], [[C1_2]]) : (!llvm.ptr, i32, i32, i32) -> () memref.prefetch %A[%i, %j], read, locality<0>, data : memref // CHECK: [[C0_2:%.*]] = llvm.mlir.constant(0 : i32) : i32 // CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : i32) : i32 // CHECK: [[C0_3:%.*]] = llvm.mlir.constant(0 : i32) : i32 -// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0_2]], [[C2]], [[C0_3]]) : (!llvm.ptr, i32, i32, i32) -> () +// CHECK: "llvm.intr.prefetch"(%{{.*}}, [[C0_2]], [[C2]], [[C0_3]]) : (!llvm.ptr, i32, i32, i32) -> () memref.prefetch %A[%i, %j], read, locality<2>, instr : memref return } @@ -234,12 +229,12 @@ func.func @prefetch(%A : memref, %i : index, %j : index) { func.func @dynamic_store(%dynamic : memref, %i : index, %j : index, %val : f32) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : f32, !llvm.ptr memref.store %val, %dynamic[%i, %j] : memref return } @@ -251,100 +246,35 @@ func.func @dynamic_store(%dynamic : memref, %i : index, %j : index, %va func.func @mixed_store(%mixed : memref<42x?xf32>, %i : index, %j : index, %val : f32) { // CHECK-DAG: %[[I:.*]] = builtin.unrealized_conversion_cast %[[Iarg]] // CHECK-DAG: %[[J:.*]] = builtin.unrealized_conversion_cast %[[Jarg]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[st0:.*]] = llvm.extractvalue %[[ld]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[offI:.*]] = llvm.mul %[[I]], %[[st0]] : i64 // CHECK-NEXT: %[[off1:.*]] = llvm.add %[[offI]], %[[J]] : i64 -// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : !llvm.ptr +// CHECK-NEXT: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK-NEXT: llvm.store %{{.*}}, %[[addr]] : f32, !llvm.ptr memref.store %val, %mixed[%i, %j] : memref<42x?xf32> return } // ----- -// CHECK-LABEL: func @memref_cast_static_to_dynamic -func.func @memref_cast_static_to_dynamic(%static : memref<10x42xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %static : memref<10x42xf32> to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_static_to_mixed -func.func @memref_cast_static_to_mixed(%static : memref<10x42xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %static : memref<10x42xf32> to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_dynamic_to_static -func.func @memref_cast_dynamic_to_static(%dynamic : memref) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %dynamic : memref to memref<10x12xf32> - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_dynamic_to_mixed -func.func @memref_cast_dynamic_to_mixed(%dynamic : memref) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %dynamic : memref to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_mixed_to_dynamic -func.func @memref_cast_mixed_to_dynamic(%mixed : memref<42x?xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %mixed : memref<42x?xf32> to memref - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_mixed_to_static -func.func @memref_cast_mixed_to_static(%mixed : memref<42x?xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %mixed : memref<42x?xf32> to memref<42x1xf32> - return -} - -// ----- - -// CHECK-LABEL: func @memref_cast_mixed_to_mixed -func.func @memref_cast_mixed_to_mixed(%mixed : memref<42x?xf32>) { -// CHECK-NOT: llvm.bitcast - %0 = memref.cast %mixed : memref<42x?xf32> to memref - return -} - -// ----- - // CHECK-LABEL: func @memref_cast_ranked_to_unranked // CHECK32-LABEL: func @memref_cast_ranked_to_unranked func.func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) { // CHECK-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> -// CHECK-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> -// CHECK-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> to !llvm.ptr +// CHECK-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> : (i64) -> !llvm.ptr +// CHECK-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>, !llvm.ptr // CHECK-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i64 -// CHECK : llvm.mlir.undef : !llvm.struct<(i64, ptr)> -// CHECK-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr)> -// CHECK-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i64, ptr)> +// CHECK : llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[p]], %{{.*}}[1] : !llvm.struct<(i64, ptr)> // CHECK32-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK32-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)> : (i64) -> !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> -// CHECK32-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> -// CHECK32-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> to !llvm.ptr +// CHECK32-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)> : (i64) -> !llvm.ptr +// CHECK32-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>, !llvm.ptr // CHECK32-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i32 -// CHECK32 : llvm.mlir.undef : !llvm.struct<(i32, ptr)> -// CHECK32-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i32, ptr)> -// CHECK32-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i32, ptr)> +// CHECK32 : llvm.mlir.undef : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[p]], %{{.*}}[1] : !llvm.struct<(i32, ptr)> %0 = memref.cast %arg : memref<42x2x?xf32> to memref<*xf32> return } @@ -353,8 +283,7 @@ func.func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) { // CHECK-LABEL: func @memref_cast_unranked_to_ranked func.func @memref_cast_unranked_to_ranked(%arg : memref<*xf32>) { -// CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)> -// CHECK-NEXT: llvm.bitcast %[[p]] : !llvm.ptr to !llvm.ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>> +// CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)> %0 = memref.cast %arg : memref<*xf32> to memref return } @@ -366,16 +295,16 @@ func.func @mixed_memref_dim(%mixed : memref<42x?x?x13x?xf32>) { // CHECK: llvm.mlir.constant(42 : index) : i64 %c0 = arith.constant 0 : index %0 = memref.dim %mixed, %c0 : memref<42x?x?x13x?xf32> -// CHECK: llvm.extractvalue %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: llvm.extractvalue %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> %c1 = arith.constant 1 : index %1 = memref.dim %mixed, %c1 : memref<42x?x?x13x?xf32> -// CHECK: llvm.extractvalue %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: llvm.extractvalue %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> %c2 = arith.constant 2 : index %2 = memref.dim %mixed, %c2 : memref<42x?x?x13x?xf32> // CHECK: llvm.mlir.constant(13 : index) : i64 %c3 = arith.constant 3 : index %3 = memref.dim %mixed, %c3 : memref<42x?x?x13x?xf32> -// CHECK: llvm.extractvalue %{{.*}}[3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: llvm.extractvalue %{{.*}}[3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> %c4 = arith.constant 4 : index %4 = memref.dim %mixed, %c4 : memref<42x?x?x13x?xf32> return @@ -389,10 +318,10 @@ func.func @memref_dim_with_dyn_index(%arg : memref<3x?xf32>, %idx : index) -> in // CHECK-DAG: %[[IDX:.*]] = builtin.unrealized_conversion_cast %[[IDXarg]] // CHECK-DAG: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK-DAG: %[[SIZES:.*]] = llvm.extractvalue %{{.*}}[3] : ![[DESCR_TY:.*]] - // CHECK-DAG: %[[SIZES_PTR:.*]] = llvm.alloca %[[C1]] x !llvm.array<2 x i64> : (i64) -> !llvm.ptr> - // CHECK-DAG: llvm.store %[[SIZES]], %[[SIZES_PTR]] : !llvm.ptr> - // CHECK-DAG: %[[RESULT_PTR:.*]] = llvm.getelementptr %[[SIZES_PTR]][0, %[[IDX]]] : (!llvm.ptr>, i64) -> !llvm.ptr - // CHECK-DAG: %[[RESULT:.*]] = llvm.load %[[RESULT_PTR]] : !llvm.ptr + // CHECK-DAG: %[[SIZES_PTR:.*]] = llvm.alloca %[[C1]] x !llvm.array<2 x i64> : (i64) -> !llvm.ptr + // CHECK-DAG: llvm.store %[[SIZES]], %[[SIZES_PTR]] : !llvm.array<2 x i64>, !llvm.ptr + // CHECK-DAG: %[[RESULT_PTR:.*]] = llvm.getelementptr %[[SIZES_PTR]][0, %[[IDX]]] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.array<2 x i64> + // CHECK-DAG: %[[RESULT:.*]] = llvm.load %[[RESULT_PTR]] : !llvm.ptr -> i64 %result = memref.dim %arg, %idx : memref<3x?xf32> return %result : index } @@ -449,13 +378,11 @@ func.func @memref_reinterpret_cast_unranked_to_dynamic_shape(%offset: index, // CHECK-DAG: [[STRIDE_1:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_1arg]] // CHECK-DAG: [[INPUT:%.*]] = builtin.unrealized_conversion_cast // CHECK: [[OUT_0:%.*]] = llvm.mlir.undef : [[TY:!.*]] -// CHECK: [[DESCRIPTOR:%.*]] = llvm.extractvalue [[INPUT]][1] : !llvm.struct<(i64, ptr)> -// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[BASE_PTR:%.*]] = llvm.load [[BASE_PTR_PTR]] : !llvm.ptr> -// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] -// CHECK-SAME: : (!llvm.ptr>) -> !llvm.ptr> -// CHECK: [[ALIGNED_PTR:%.*]] = llvm.load [[ALIGNED_PTR_PTR]] : !llvm.ptr> +// CHECK: [[DESCRIPTOR:%.*]] = llvm.extractvalue [[INPUT]][1] : !llvm.struct<(i64, ptr)> +// CHECK: [[BASE_PTR:%.*]] = llvm.load [[DESCRIPTOR]] : !llvm.ptr -> !llvm.ptr +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[DESCRIPTOR]]{{\[}}1] +// CHECK-SAME: : (!llvm.ptr) -> !llvm.ptr, !llvm.ptr +// CHECK: [[ALIGNED_PTR:%.*]] = llvm.load [[ALIGNED_PTR_PTR]] : !llvm.ptr -> !llvm.ptr // CHECK: [[OUT_1:%.*]] = llvm.insertvalue [[BASE_PTR]], [[OUT_0]][0] : [[TY]] // CHECK: [[OUT_2:%.*]] = llvm.insertvalue [[ALIGNED_PTR]], [[OUT_1]][1] : [[TY]] // CHECK: [[OUT_3:%.*]] = llvm.insertvalue [[OFFSET]], [[OUT_2]][2] : [[TY]] @@ -475,8 +402,8 @@ func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { // CHECK: [[INPUT:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[INPUT_TY:!.*]] // CHECK: [[SHAPE:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[SHAPE_TY:!.*]] // CHECK: [[RANK:%.*]] = llvm.extractvalue [[SHAPE]][3, 0] : [[SHAPE_TY]] -// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> -// CHECK: [[UNRANKED_OUT_1:%.*]] = llvm.insertvalue [[RANK]], [[UNRANKED_OUT_O]][0] : !llvm.struct<(i64, ptr)> +// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK: [[UNRANKED_OUT_1:%.*]] = llvm.insertvalue [[RANK]], [[UNRANKED_OUT_O]][0] : !llvm.struct<(i64, ptr)> // Compute size in bytes to allocate result ranked descriptor // CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : index) : i64 @@ -492,21 +419,14 @@ func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { // CHECK: [[ALLOC_PTR:%.*]] = llvm.extractvalue [[INPUT]][0] : [[INPUT_TY]] // CHECK: [[ALIGN_PTR:%.*]] = llvm.extractvalue [[INPUT]][1] : [[INPUT_TY]] // CHECK: [[OFFSET:%.*]] = llvm.extractvalue [[INPUT]][2] : [[INPUT_TY]] -// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] -// CHECK-SAME: !llvm.ptr to !llvm.ptr> -// CHECK: llvm.store [[ALLOC_PTR]], [[BASE_PTR_PTR]] : !llvm.ptr> -// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] -// CHECK: llvm.store [[ALIGN_PTR]], [[ALIGNED_PTR_PTR]] : !llvm.ptr> -// CHECK: [[BASE_PTR_PTR__:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> -// CHECK: [[OFFSET_PTR_:%.*]] = llvm.getelementptr [[BASE_PTR_PTR__]]{{\[}}2] -// CHECK: [[OFFSET_PTR:%.*]] = llvm.bitcast [[OFFSET_PTR_]] -// CHECK: llvm.store [[OFFSET]], [[OFFSET_PTR]] : !llvm.ptr +// CHECK: llvm.store [[ALLOC_PTR]], [[UNDERLYING_DESC]] : !llvm.ptr, !llvm.ptr +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[UNDERLYING_DESC]]{{\[}}1] +// CHECK: llvm.store [[ALIGN_PTR]], [[ALIGNED_PTR_PTR]] : !llvm.ptr, !llvm.ptr +// CHECK: [[OFFSET_PTR:%.*]] = llvm.getelementptr [[UNDERLYING_DESC]]{{\[}}2] +// CHECK: llvm.store [[OFFSET]], [[OFFSET_PTR]] : i64, !llvm.ptr // Iterate over shape operand in reverse order and set sizes and strides. -// CHECK: [[STRUCT_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] -// CHECK-SAME: !llvm.ptr to !llvm.ptr, ptr, i64, i64)>> -// CHECK: [[SIZES_PTR:%.*]] = llvm.getelementptr [[STRUCT_PTR]]{{\[}}0, 3] +// CHECK: [[SIZES_PTR:%.*]] = llvm.getelementptr [[UNDERLYING_DESC]]{{\[}}0, 3] // CHECK: [[STRIDES_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[RANK]]] // CHECK: [[SHAPE_IN_PTR:%.*]] = llvm.extractvalue [[SHAPE]][1] : [[SHAPE_TY]] // CHECK: [[C1_:%.*]] = llvm.mlir.constant(1 : index) : i64 @@ -520,11 +440,11 @@ func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { // CHECK: ^bb2: // CHECK: [[SIZE_PTR:%.*]] = llvm.getelementptr [[SHAPE_IN_PTR]]{{\[}}[[DIM]]] -// CHECK: [[SIZE:%.*]] = llvm.load [[SIZE_PTR]] : !llvm.ptr +// CHECK: [[SIZE:%.*]] = llvm.load [[SIZE_PTR]] : !llvm.ptr -> i64 // CHECK: [[TARGET_SIZE_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[DIM]]] -// CHECK: llvm.store [[SIZE]], [[TARGET_SIZE_PTR]] : !llvm.ptr +// CHECK: llvm.store [[SIZE]], [[TARGET_SIZE_PTR]] : i64, !llvm.ptr // CHECK: [[TARGET_STRIDE_PTR:%.*]] = llvm.getelementptr [[STRIDES_PTR]]{{\[}}[[DIM]]] -// CHECK: llvm.store [[CUR_STRIDE]], [[TARGET_STRIDE_PTR]] : !llvm.ptr +// CHECK: llvm.store [[CUR_STRIDE]], [[TARGET_STRIDE_PTR]] : i64, !llvm.ptr // CHECK: [[UPDATE_STRIDE:%.*]] = llvm.mul [[CUR_STRIDE]], [[SIZE]] : i64 // CHECK: [[STRIDE_COND:%.*]] = llvm.sub [[DIM]], [[C1_]] : i64 // CHECK: llvm.br ^bb1([[STRIDE_COND]], [[UPDATE_STRIDE]] : i64, i64) @@ -547,11 +467,8 @@ func.func @memref_of_memref() { // Check that the types are converted as expected. // ALIGNED-ALLOC: llvm.call @aligned_alloc - // ALIGNED-ALLOC: llvm.bitcast %{{.*}} : !llvm.ptr to - // ALIGNED-ALLOC-SAME: !llvm. - // ALIGNED-ALLOC-SAME: [[INNER:ptr, ptr, i64, array<1 x i64>, array<1 x i64>\)>>]] // ALIGNED-ALLOC: llvm.mlir.undef - // ALIGNED-ALLOC-SAME: !llvm.struct<([[INNER]], [[INNER]], i64, array<1 x i64>, array<1 x i64>)> + // ALIGNED-ALLOC-SAME: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %0 = memref.alloc() : memref<1xmemref<1xf32>> return } @@ -572,11 +489,8 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry> } { // Check that the types are converted as expected. // ALIGNED-ALLOC: llvm.call @aligned_alloc - // ALIGNED-ALLOC: llvm.bitcast %{{.*}} : !llvm.ptr to - // ALIGNED-ALLOC-SAME: !llvm. - // ALIGNED-ALLOC-SAME: [[INNER:ptr, ptr, i32, array<1 x i32>, array<1 x i32>\)>>]] // ALIGNED-ALLOC: llvm.mlir.undef - // ALIGNED-ALLOC-SAME: !llvm.struct<([[INNER]], [[INNER]], i32, array<1 x i32>, array<1 x i32>)> + // ALIGNED-ALLOC-SAME: !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)> %0 = memref.alloc() : memref<1xmemref<1xf32>> return } @@ -588,13 +502,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry> } { // ALIGNED-ALLOC-LABEL: @memref_of_memref_of_memref func.func @memref_of_memref_of_memref() { // Sizeof computation is as usual, also check the type. - // ALIGNED-ALLOC: %[[NULL:.*]] = llvm.mlir.null : !llvm.ptr< - // ALIGNED-ALLOC-SAME: struct<( - // ALIGNED-ALLOC-SAME: [[INNER:ptr, ptr, i64, array<1 x i64>, array<1 x i64>\)>>]], - // ALIGNED-ALLOC-SAME: [[INNER]], - // ALIGNED-ALLOC-SAME: i64, array<1 x i64>, array<1 x i64> - // ALIGNED-ALLOC-SAME: )> - // ALIGNED-ALLOC-SAME: > + // ALIGNED-ALLOC: %[[NULL:.*]] = llvm.mlir.null : !llvm.ptr // ALIGNED-ALLOC: %[[PTR:.*]] = llvm.getelementptr // ALIGNED-ALLOC: %[[SIZEOF:.*]] = llvm.ptrtoint @@ -611,7 +519,7 @@ func.func @memref_of_memref_of_memref() { // ALIGNED-ALLOC-LABEL: @ranked_unranked func.func @ranked_unranked() { // ALIGNED-ALLOC: llvm.mlir.null - // ALIGNED-ALLOC-SAME: !llvm.[[INNER:ptr\)>>]] + // ALIGNED-ALLOC-SAME: !llvm.ptr // ALIGNED-ALLOC: llvm.getelementptr // ALIGNED-ALLOC: llvm.ptrtoint @@ -619,8 +527,6 @@ func.func @ranked_unranked() { // sizeof(pointer)) = 16. // ALIGNED-ALLOC: llvm.mlir.constant(16 : index) // ALIGNED-ALLOC: llvm.call @aligned_alloc - // ALIGNED-ALLOC: llvm.bitcast - // ALIGNED-ALLOC-SAME: !llvm.ptr to !llvm.[[INNER]] %0 = memref.alloc() : memref<1 x memref<* x f32>> memref.cast %0 : memref<1 x memref<* x f32>> to memref<* x memref<* x f32>> return @@ -638,25 +544,21 @@ func.func @realloc_dynamic(%in: memref, %d: index) -> memref{ // CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] : i64 // CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] // CHECK: ^bb1: -// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[dst_size]]) -// CHECK: %[[new_buffer:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_raw]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor]][0] -// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor_update1]][1] +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] +// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor_update1]][1] // CHECK: llvm.br ^bb2(%[[descriptor_update2]] -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref @@ -680,34 +582,30 @@ func.func @realloc_dynamic_alignment(%in: memref, %d: index) -> memref +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[drc_dim]], %[[dst_es]] // CHECK: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 // CHECK: %[[adjust_dst_size:.*]] = llvm.add %[[dst_size]], %[[alignment]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[adjust_dst_size]]) -// CHECK: %[[new_buffer_unaligned:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr -// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_unaligned]] : !llvm.ptr +// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_raw]] : !llvm.ptr // CHECK: %[[const_1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[alignment_m1:.*]] = llvm.sub %[[alignment]], %[[const_1]] // CHECK: %[[ptr_alignment_m1:.*]] = llvm.add %[[new_buffer_int]], %[[alignment_m1]] // CHECK: %[[padding:.*]] = llvm.urem %[[ptr_alignment_m1]], %[[alignment]] // CHECK: %[[new_buffer_aligned_int:.*]] = llvm.sub %[[ptr_alignment_m1]], %[[padding]] -// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr +// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_aligned]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_unaligned]], %[[descriptor]][0] +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] // CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor_update1]][1] // CHECK: llvm.br ^bb2(%[[descriptor_update2]] -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref @@ -718,9 +616,9 @@ func.func @realloc_dynamic_alignment(%in: memref, %d: index) -> memref +// ALIGNED-ALLOC: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // ALIGNED-ALLOC: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// ALIGNED-ALLOC: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// ALIGNED-ALLOC: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // ALIGNED-ALLOC: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // ALIGNED-ALLOC: %[[src_size:.*]] = llvm.mul %[[drc_dim]], %[[dst_es]] // ALIGNED-ALLOC-DAG: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 @@ -730,19 +628,15 @@ func.func @realloc_dynamic_alignment(%in: memref, %d: index) -> memref to !llvm.ptr // ALIGNED-ALLOC: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // ALIGNED-ALLOC: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// ALIGNED-ALLOC-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// ALIGNED-ALLOC-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// ALIGNED-ALLOC: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// ALIGNED-ALLOC: "llvm.intr.memcpy"(%[[new_buffer_raw]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // ALIGNED-ALLOC: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// ALIGNED-ALLOC: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// ALIGNED-ALLOC: llvm.call @free(%[[old_buffer_unaligned_void]]) -// ALIGNED-ALLOC: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor]][0] -// ALIGNED-ALLOC: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor_update1]][1] -// ALIGNED-ALLOC: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) -// ALIGNED-ALLOC: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// ALIGNED-ALLOC: llvm.call @free(%[[old_buffer_unaligned]]) +// ALIGNED-ALLOC: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] +// ALIGNED-ALLOC: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor_update1]][1] +// ALIGNED-ALLOC: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) +// ALIGNED-ALLOC: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // ALIGNED-ALLOC: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // ALIGNED-ALLOC: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // ALIGNED-ALLOC: return %[[descriptor_update5]] : memref diff --git a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir index 023955987fb5..d9552972e2d9 100644 --- a/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/convert-static-memref-ops.mlir @@ -1,18 +1,17 @@ -// RUN: mlir-opt -convert-memref-to-llvm -split-input-file %s | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=1' -split-input-file %s | FileCheck %s // CHECK-LABEL: func @zero_d_alloc() func.func @zero_d_alloc() -> memref { // CHECK: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK: llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr -// CHECK: %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> -// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[one]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: %[[ptr:.*]] = llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr +// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> // CHECK: unrealized_conversion_cast %{{.*}} %0 = memref.alloc() : memref @@ -24,9 +23,8 @@ func.func @zero_d_alloc() -> memref { // CHECK-LABEL: func @zero_d_dealloc func.func @zero_d_dealloc(%arg0: memref) { // CHECK: unrealized_conversion_cast -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[bc]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return @@ -38,25 +36,24 @@ func.func @zero_d_dealloc(%arg0: memref) { func.func @aligned_1d_alloc() -> memref<42xf32> { // CHECK: %[[sz1:.*]] = llvm.mlir.constant(42 : index) : i64 // CHECK: %[[st1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 // CHECK: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 // CHECK: %[[allocsize:.*]] = llvm.add %[[size_bytes]], %[[alignment]] : i64 -// CHECK: %[[allocated:.*]] = llvm.call @malloc(%[[allocsize]]) : (i64) -> !llvm.ptr -// CHECK: %[[ptr:.*]] = llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr -// CHECK: %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[ptr]] : !llvm.ptr to i64 +// CHECK: %[[ptr:.*]] = llvm.call @malloc(%[[allocsize]]) : (i64) -> !llvm.ptr +// CHECK: %[[allocatedAsInt:.*]] = llvm.ptrtoint %[[ptr]] : !llvm.ptr to i64 // CHECK: %[[one_1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[bump:.*]] = llvm.sub %[[alignment]], %[[one_1]] : i64 // CHECK: %[[bumped:.*]] = llvm.add %[[allocatedAsInt]], %[[bump]] : i64 // CHECK: %[[mod:.*]] = llvm.urem %[[bumped]], %[[alignment]] : i64 // CHECK: %[[aligned:.*]] = llvm.sub %[[bumped]], %[[mod]] : i64 -// CHECK: %[[alignedBitCast:.*]] = llvm.inttoptr %[[aligned]] : i64 to !llvm.ptr -// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> -// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> -// CHECK: llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: %[[alignedBitCast:.*]] = llvm.inttoptr %[[aligned]] : i64 to !llvm.ptr +// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: llvm.insertvalue %[[ptr]], %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: llvm.insertvalue %[[alignedBitCast]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: llvm.insertvalue %[[c0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %0 = memref.alloc() {alignment = 8} : memref<42xf32> return %0 : memref<42xf32> } @@ -66,11 +63,10 @@ func.func @aligned_1d_alloc() -> memref<42xf32> { // CHECK-LABEL: func @static_alloc() func.func @static_alloc() -> memref<32x18xf32> { // CHECK: %[[num_elems:.*]] = llvm.mlir.constant(576 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK: %[[allocated:.*]] = llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr -// CHECK: llvm.bitcast %[[allocated]] : !llvm.ptr to !llvm.ptr +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: llvm.call @malloc(%[[size_bytes]]) : (i64) -> !llvm.ptr %0 = memref.alloc() : memref<32x18xf32> return %0 : memref<32x18xf32> } @@ -83,19 +79,19 @@ func.func @static_alloca() -> memref<32x18xf32> { // CHECK: %[[sz2:.*]] = llvm.mlir.constant(18 : index) : i64 // CHECK: %[[st2:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[num_elems:.*]] = llvm.mlir.constant(576 : index) : i64 -// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr -// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 -// CHECK: %[[allocated:.*]] = llvm.alloca %[[size_bytes]] x f32 : (i64) -> !llvm.ptr +// CHECK: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[num_elems]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: %[[size_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK: %[[allocated:.*]] = llvm.alloca %[[size_bytes]] x f32 : (i64) -> !llvm.ptr %0 = memref.alloca() : memref<32x18xf32> // Test with explicitly specified alignment. llvm.alloca takes care of the // alignment. The same pointer is thus used for allocation and aligned // accesses. - // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr - // CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[alloca_aligned:.*]] = llvm.alloca %{{.*}} x f32 {alignment = 32 : i64} : (i64) -> !llvm.ptr + // CHECK: %[[desc:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[desc1:.*]] = llvm.insertvalue %[[alloca_aligned]], %[[desc]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[alloca_aligned]], %[[desc1]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> memref.alloca() {alignment = 32} : memref<32x18xf32> return %0 : memref<32x18xf32> } @@ -104,9 +100,8 @@ func.func @static_alloca() -> memref<32x18xf32> { // CHECK-LABEL: func @static_dealloc func.func @static_dealloc(%static: memref<10x8xf32>) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[bc:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[bc]]) : (!llvm.ptr) -> () +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: llvm.call @free(%[[ptr]]) : (!llvm.ptr) -> () memref.dealloc %static : memref<10x8xf32> return } @@ -115,8 +110,8 @@ func.func @static_dealloc(%static: memref<10x8xf32>) { // CHECK-LABEL: func @zero_d_load func.func @zero_d_load(%arg0: memref) -> f32 { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %{{.*}} = llvm.load %[[ptr]] : !llvm.ptr +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %{{.*}} = llvm.load %[[ptr]] : !llvm.ptr -> f32 %0 = memref.load %arg0[] : memref return %0 : f32 } @@ -130,12 +125,12 @@ func.func @zero_d_load(%arg0: memref) -> f32 { func.func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) { // CHECK: %[[II:.*]] = builtin.unrealized_conversion_cast %[[I]] // CHECK: %[[JJ:.*]] = builtin.unrealized_conversion_cast %[[J]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[st0:.*]] = llvm.mlir.constant(42 : index) : i64 // CHECK: %[[offI:.*]] = llvm.mul %[[II]], %[[st0]] : i64 // CHECK: %[[off1:.*]] = llvm.add %[[offI]], %[[JJ]] : i64 -// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: llvm.load %[[addr]] : !llvm.ptr +// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: llvm.load %[[addr]] : !llvm.ptr -> f32 %0 = memref.load %static[%i, %j] : memref<10x42xf32> return } @@ -144,8 +139,8 @@ func.func @static_load(%static : memref<10x42xf32>, %i : index, %j : index) { // CHECK-LABEL: func @zero_d_store func.func @zero_d_store(%arg0: memref, %arg1: f32) { -// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: llvm.store %{{.*}}, %[[ptr]] : !llvm.ptr +// CHECK: %[[ptr:.*]] = llvm.extractvalue %[[ld:.*]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: llvm.store %{{.*}}, %[[ptr]] : f32, !llvm.ptr memref.store %arg1, %arg0[] : memref return } @@ -158,12 +153,12 @@ func.func @zero_d_store(%arg0: memref, %arg1: f32) { func.func @static_store(%static : memref<10x42xf32>, %i : index, %j : index, %val : f32) { // CHECK: %[[II:.*]] = builtin.unrealized_conversion_cast %[[I]] // CHECK: %[[JJ:.*]] = builtin.unrealized_conversion_cast %[[J]] -// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[st0:.*]] = llvm.mlir.constant(42 : index) : i64 // CHECK: %[[offI:.*]] = llvm.mul %[[II]], %[[st0]] : i64 // CHECK: %[[off1:.*]] = llvm.add %[[offI]], %[[JJ]] : i64 -// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: llvm.store %{{.*}}, %[[addr]] : !llvm.ptr +// CHECK: %[[addr:.*]] = llvm.getelementptr %[[ptr]][%[[off1]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 +// CHECK: llvm.store %{{.*}}, %[[addr]] : f32, !llvm.ptr memref.store %val, %static[%i, %j] : memref<10x42xf32> return @@ -203,11 +198,11 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry> } { // CHECK: %[[CST:.*]] = builtin.unrealized_conversion_cast // CHECK: llvm.mlir.null // CHECK: llvm.getelementptr %{{.*}}[[CST]] - // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr<{{.*}}> to i32 - // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr<{{.*}}> to i32 + // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr to i32 + // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr to i32 // CHECK: llvm.add %{{.*}} : i32 // CHECK: llvm.call @malloc(%{{.*}}) : (i32) -> !llvm.ptr - // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr<{{.*}}> to i32 + // CHECK: llvm.ptrtoint %{{.*}} : !llvm.ptr to i32 // CHECK: llvm.sub {{.*}} : i32 // CHECK: llvm.add {{.*}} : i32 // CHECK: llvm.urem {{.*}} : i32 @@ -224,34 +219,34 @@ memref.global "private" constant @__constant_3xi64 : memref<3xi64> = dense<[2, 6 // CHECK-LABEL: func @memref.reshape // CHECK-SAME: %[[arg0:.*]]: memref<4x5x6xf32>) -> memref<2x6x20xf32> func.func @memref.reshape(%arg0: memref<4x5x6xf32>) -> memref<2x6x20xf32> { - // CHECK: %[[cast0:.*]] = builtin.unrealized_conversion_cast %arg0 : memref<4x5x6xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[cast0:.*]] = builtin.unrealized_conversion_cast %arg0 : memref<4x5x6xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> %0 = memref.get_global @__constant_3xi64 : memref<3xi64> - // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[elem0:.*]] = llvm.extractvalue %[[cast0]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[elem1:.*]] = llvm.extractvalue %[[cast0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[elem0]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[elem1]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[elem0:.*]] = llvm.extractvalue %[[cast0]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[elem1:.*]] = llvm.extractvalue %[[cast0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[elem0]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[elem1]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[zero:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[twenty0:.*]] = llvm.mlir.constant(20 : index) : i64 - // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[twenty0]], %[[insert2]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one]], %[[insert3]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[twenty0]], %[[insert2]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one]], %[[insert3]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[twenty1:.*]] = llvm.mlir.constant(20 : index) : i64 // CHECK: %[[six:.*]] = llvm.mlir.constant(6 : index) : i64 - // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[six]], %[[insert4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[twenty1]], %[[insert5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[six]], %[[insert4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[twenty1]], %[[insert5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[hundred_and_twenty:.*]] = llvm.mlir.constant(120 : index) : i64 // CHECK: %[[two:.*]] = llvm.mlir.constant(2 : index) : i64 - // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[two]], %[[insert6]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[hundred_and_twenty]], %[[insert7]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[two]], %[[insert6]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[hundred_and_twenty]], %[[insert7]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[cast1:.*]] = builtin.unrealized_conversion_cast %[[insert8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<2x6x20xf32> + // CHECK: %[[cast1:.*]] = builtin.unrealized_conversion_cast %[[insert8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<2x6x20xf32> %1 = memref.reshape %arg0(%0) : (memref<4x5x6xf32>, memref<3xi64>) -> memref<2x6x20xf32> // CHECK: return %[[cast1]] : memref<2x6x20xf32> @@ -263,44 +258,44 @@ func.func @memref.reshape(%arg0: memref<4x5x6xf32>) -> memref<2x6x20xf32> { // CHECK-LABEL: func @memref.reshape.dynamic.dim // CHECK-SAME: %[[arg:.*]]: memref, %[[shape:.*]]: memref<4xi64>) -> memref func.func @memref.reshape.dynamic.dim(%arg: memref, %shape: memref<4xi64>) -> memref { - // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<4xi64> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> - // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0]][1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<4xi64> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef]][0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0]][1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[zero0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1]][2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[one0:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[thirty_two0:.*]] = llvm.mlir.constant(32 : index) : i64 - // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[thirty_two0]], %[[insert2]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0]], %[[insert3]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[thirty_two0]], %[[insert2]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0]], %[[insert3]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[thirty_two1:.*]] = llvm.mlir.constant(32 : index) : i64 // CHECK: %[[twelve:.*]] = llvm.mlir.constant(12 : index) : i64 - // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[twelve]], %[[insert4]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[thirty_two1]], %[[insert5]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert5:.*]] = llvm.insertvalue %[[twelve]], %[[insert4]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert6:.*]] = llvm.insertvalue %[[thirty_two1]], %[[insert5]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[three_hundred_and_eighty_four:.*]] = llvm.mlir.constant(384 : index) : i64 // CHECK: %[[one1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0]][%[[one1]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0]] : !llvm.ptr - // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[shape_load0]], %[[insert6]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[three_hundred_and_eighty_four]], %[[insert7]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0]][%[[one1]]] : (!llvm.ptr, i64) -> !llvm.ptr, i64 + // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0]] : !llvm.ptr -> i64 + // CHECK: %[[insert7:.*]] = llvm.insertvalue %[[shape_load0]], %[[insert6]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert8:.*]] = llvm.insertvalue %[[three_hundred_and_eighty_four]], %[[insert7]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> // CHECK: %[[mul:.*]] = llvm.mul %19, %23 : i64 // CHECK: %[[zero1:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[shape_ptr1:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[shape_gep1:.*]] = llvm.getelementptr %[[shape_ptr1]][%[[zero1]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[shape_load1:.*]] = llvm.load %[[shape_gep1]] : !llvm.ptr - // CHECK: %[[insert9:.*]] = llvm.insertvalue %[[shape_load1]], %[[insert8]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[insert10:.*]] = llvm.insertvalue %[[mul]], %[[insert9]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[shape_ptr1:.*]] = llvm.extractvalue %[[shape_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_gep1:.*]] = llvm.getelementptr %[[shape_ptr1]][%[[zero1]]] : (!llvm.ptr, i64) -> !llvm.ptr, i64 + // CHECK: %[[shape_load1:.*]] = llvm.load %[[shape_gep1]] : !llvm.ptr -> i64 + // CHECK: %[[insert9:.*]] = llvm.insertvalue %[[shape_load1]], %[[insert8]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> + // CHECK: %[[insert10:.*]] = llvm.insertvalue %[[mul]], %[[insert9]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> - // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert10]] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> to memref + // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert10]] : !llvm.struct<(ptr, ptr, i64, array<4 x i64>, array<4 x i64>)> to memref %0 = memref.reshape %arg(%shape) : (memref, memref<4xi64>) -> memref return %0 : memref @@ -312,27 +307,27 @@ func.func @memref.reshape.dynamic.dim(%arg: memref, %shape: memref<4x // CHECK-LABEL: func @memref.reshape_index // CHECK-SAME: %[[arg:.*]]: memref, %[[shape:.*]]: memref<1xindex> func.func @memref.reshape_index(%arg0: memref, %shape: memref<1xindex>) -> memref { - // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<1xindex> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef:.*]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[arg_cast:.*]] = builtin.unrealized_conversion_cast %[[arg]] : memref to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[shape_cast:.*]] = builtin.unrealized_conversion_cast %[[shape]] : memref<1xindex> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[alloc_ptr:.*]] = llvm.extractvalue %[[arg_cast]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[align_ptr:.*]] = llvm.extractvalue %[[arg_cast]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[insert0:.*]] = llvm.insertvalue %[[alloc_ptr]], %[[undef:.*]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[insert1:.*]] = llvm.insertvalue %[[align_ptr]], %[[insert0:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[zero0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1:.*]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[insert2:.*]] = llvm.insertvalue %[[zero0]], %[[insert1:.*]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[one0:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[zero1:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0:.*]][%[[zero1:.*]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0:.*]] : !llvm.ptr - // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[shape_load0:.*]], %[[insert2:.*]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0:.*]], %[[insert3:.*]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_ptr0:.*]] = llvm.extractvalue %[[shape_cast:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[shape_gep0:.*]] = llvm.getelementptr %[[shape_ptr0:.*]][%[[zero1:.*]]] : (!llvm.ptr, i64) -> !llvm.ptr, i64 + // CHECK: %[[shape_load0:.*]] = llvm.load %[[shape_gep0:.*]] : !llvm.ptr -> i64 + // CHECK: %[[insert3:.*]] = llvm.insertvalue %[[shape_load0:.*]], %[[insert2:.*]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[insert4:.*]] = llvm.insertvalue %[[one0:.*]], %[[insert3:.*]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert4:.*]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref + // CHECK: %[[result_cast:.*]] = builtin.unrealized_conversion_cast %[[insert4:.*]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref // CHECK: return %[[result_cast:.*]] : memref %1 = memref.reshape %arg0(%shape) : (memref, memref<1xindex>) -> memref @@ -344,31 +339,27 @@ func.func @memref.reshape_index(%arg0: memref, %shape: memref<1xindex>) // CHECK-LABEL: func.func @realloc_static( // CHECK-SAME: %[[arg0:.*]]: memref<2xi32>) -> memref<4xi32> { func.func @realloc_static(%in: memref<2xi32>) -> memref<4xi32>{ -// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xi32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xi32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[src_dim:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[dst_dim:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] // CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] // CHECK: ^bb1: -// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[dst_size]]) -// CHECK: %[[new_buffer:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_raw]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor]][0] -// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor_update1]][1] -// CHECK: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] +// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor_update1]][1] +// CHECK: llvm.br ^bb2(%[[descriptor_update2]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>) +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref<4xi32> @@ -382,40 +373,36 @@ func.func @realloc_static(%in: memref<2xi32>) -> memref<4xi32>{ // CHECK-LABEL: func.func @realloc_static_alignment( // CHECK-SAME: %[[arg0:.*]]: memref<2xf32>) -> memref<4xf32> { func.func @realloc_static_alignment(%in: memref<2xf32>) -> memref<4xf32>{ -// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xf32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> +// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] : memref<2xf32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[src_dim:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[dst_dim:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] : i64 // CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] // CHECK: ^bb1: -// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr // CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] -// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 // CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] // CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] // CHECK: %[[alignment:.*]] = llvm.mlir.constant(8 : index) : i64 // CHECK: %[[adjust_dst_size:.*]] = llvm.add %[[dst_size]], %[[alignment]] // CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[adjust_dst_size]]) -// CHECK: %[[new_buffer_unaligned:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr -// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_unaligned]] : !llvm.ptr +// CHECK: %[[new_buffer_int:.*]] = llvm.ptrtoint %[[new_buffer_raw]] : !llvm.ptr // CHECK: %[[const_1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[alignment_m1:.*]] = llvm.sub %[[alignment]], %[[const_1]] // CHECK: %[[ptr_alignment_m1:.*]] = llvm.add %[[new_buffer_int]], %[[alignment_m1]] // CHECK: %[[padding:.*]] = llvm.urem %[[ptr_alignment_m1]], %[[alignment]] // CHECK: %[[new_buffer_aligned_int:.*]] = llvm.sub %[[ptr_alignment_m1]], %[[padding]] -// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr +// CHECK: %[[new_buffer_aligned:.*]] = llvm.inttoptr %[[new_buffer_aligned_int]] : i64 to !llvm.ptr // CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] // CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 -// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr -// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_aligned]], %[[old_buffer_aligned]], %[[src_size]], %[[volatile]]) // CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] -// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr -// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) -// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_unaligned]], %[[descriptor]][0] +// CHECK: llvm.call @free(%[[old_buffer_unaligned]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer_raw]], %[[descriptor]][0] // CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer_aligned]], %[[descriptor_update1]][1] // CHECK: llvm.br ^bb2(%[[descriptor_update2]] -// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): // CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] // CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] // CHECK: return %[[descriptor_update5]] : memref<4xf32> diff --git a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir index 51650adc9a9f..7a21bf36b853 100644 --- a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -expand-strided-metadata -convert-memref-to-llvm -lower-affine -convert-arith-to-llvm -cse %s -split-input-file | FileCheck %s +// RUN: mlir-opt -expand-strided-metadata -finalize-memref-to-llvm='use-opaque-pointers=1' -lower-affine -convert-arith-to-llvm -cse %s -split-input-file | FileCheck %s // // This test demonstrates a full "memref to llvm" pipeline where // we first expand some of the memref operations (using affine, @@ -7,7 +7,7 @@ // // Note: We run CSE in that test to get rid of duplicated // unrealized_conversion_cast that are inserted with -// convert-memref-to-llvm and then convert-arith-to-llvm. +// finalize-memref-to-llvm and then convert-arith-to-llvm. // The final code is still not perfect, because we have // noop unrealized_conversion_cast from i64 to index // and back. @@ -56,8 +56,8 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 // CHECK: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -65,14 +65,14 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -92,8 +92,8 @@ func.func @subview_non_zero_addrspace(%0 : memref<64x4xf32, strided<[4, 1], offs // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -101,14 +101,14 @@ func.func @subview_non_zero_addrspace(%0 : memref<64x4xf32, strided<[4, 1], offs // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] : memref<64x4xf32, strided<[4, 1], offset: 0>, 3> @@ -128,8 +128,8 @@ func.func @subview_const_size(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[C4]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -137,15 +137,15 @@ func.func @subview_const_size(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, // CHECK: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE1:.*]] = llvm.mlir.constant(2 : index) : i64 - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][4, 2][%arg0, %arg1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -165,22 +165,22 @@ func.func @subview_const_stride(%0 : memref<64x4xf32, strided<[4, 1], offset: 0> // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0f]] // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[OFF0:.*]] = llvm.mul %[[ARG0]], %[[C4]] : i64 // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF0]], %[[ARG1]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C4]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C4]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(2 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][1, 2] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -196,21 +196,21 @@ func.func @subview_const_stride_and_offset(%0 : memref<64x4xf32, strided<[4, 1], // The last "insertvalue" that populates the memref descriptor from the function arguments. // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_OFF:.*]] = llvm.mlir.constant(8 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(62 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE1:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[0, 8][62, 3][1, 1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -231,8 +231,8 @@ func.func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, strided<[4, 1], of // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1f]] // CHECK-DAG: %[[ARG2:.*]] = builtin.unrealized_conversion_cast %[[ARG2f]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64 // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[DESCSTRIDE0]] : i64 to index @@ -242,16 +242,16 @@ func.func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, strided<[4, 1], of // CHECK: %[[OFF2:.*]] = llvm.add %[[OFF0]], %[[BASE_OFF]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF2]] : i64 to index // CHECK: %[[OFF2:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(62 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0_V2]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.subview %0[%arg1, 8][62, %arg2][%arg0, 1] : memref<64x4xf32, strided<[4, 1], offset: 0>> @@ -266,23 +266,23 @@ func.func @subview_mixed_static_dynamic(%0 : memref<64x4xf32, strided<[4, 1], of func.func @subview_leading_operands(%0 : memref<5x3xf32>, %1: memref<5x?xf32>) -> memref<3x3xf32, strided<[3, 1], offset: 6>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] // Alloc ptr - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 // Aligned ptr - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Offset // CHECK: %[[CST_OFF:.*]] = llvm.mlir.constant(6 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 0: both static extracted from type. // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C3]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C3]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 1: both static. - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %2 = memref.subview %0[2, 0][3, 3][1, 1]: memref<5x3xf32> to memref<3x3xf32, strided<[3, 1], offset: 6>> return %2 : memref<3x3xf32, strided<[3, 1], offset: 6>> @@ -294,30 +294,30 @@ func.func @subview_leading_operands(%0 : memref<5x3xf32>, %1: memref<5x?xf32>) - // CHECK: %[[MEM:[a-zA-Z0-9]*]]: memref func.func @subview_leading_operands_dynamic(%0 : memref<5x?xf32>) -> memref<3x?xf32, strided<[?, 1], offset: ?>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEMREF]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEMREF]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 // Extract strides - // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Compute and insert offset from 2 + dynamic value. // CHECK: %[[CST_OFF0:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[OFF0:.*]] = llvm.mul %[[STRIDE0]], %[[CST_OFF0]] : i64 // CHECK: %[[TMP:.*]] = builtin.unrealized_conversion_cast %[[OFF0]] : i64 to index // CHECK: %[[OFF0:.*]] = builtin.unrealized_conversion_cast %[[TMP]] : index to i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Alloc ptr - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Aligned ptr - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 0: both static. // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // Sizes and strides @rank 1: static stride 1, dynamic size unchanged from source memref. - // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[CST_STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[CST_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %c0 = arith.constant 1 : index %d0 = memref.dim %0, %c0 : memref<5x?xf32> @@ -331,19 +331,19 @@ func.func @subview_leading_operands_dynamic(%0 : memref<5x?xf32>) -> memref<3x?x // CHECK: %[[MEM:.*]]: memref func.func @subview_rank_reducing_leading_operands(%0 : memref<5x3xf32>) -> memref<3xf32, strided<[1], offset: 3>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // Alloc ptr - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // Aligned ptr - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C3]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C3]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // Sizes and strides @rank 0: both static. - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_STRIDE0:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %1 = memref.subview %0[1, 0][1, 3][1, 1]: memref<5x3xf32> to memref<3xf32, strided<[1], offset: 3>> @@ -356,18 +356,18 @@ func.func @subview_rank_reducing_leading_operands(%0 : memref<5x3xf32>) -> memre // CHECK-SAME: (%[[MEM:.*]]: memref<7xf32>) func.func @subview_negative_stride(%arg0 : memref<7xf32>) -> memref<7xf32, strided<[-1], offset: 6>> { // CHECK: %[[MEMREF:.*]] = builtin.unrealized_conversion_cast %[[MEM]] - // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 - // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEMREF]][0] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[BASE_ALIGNED:.*]] = llvm.extractvalue %[[MEMREF]][1] : !llvm.struct<(ptr, ptr, i64 + // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_OFF0:.*]] = llvm.mlir.constant(6 : index) : i64 - // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[CST_OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_SIZE0:.*]] = llvm.mlir.constant(7 : index) : i64 - // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[CST_SIZE0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[CST_STRIDE0:.*]] = llvm.mlir.constant(-1 : index) : i64 - // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC4]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref<7xf32, strided<[-1], offset: 6>> + // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[CST_STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC4]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> to memref<7xf32, strided<[-1], offset: 6>> // CHECK: return %[[RES]] : memref<7xf32, strided<[-1], offset: 6>> %0 = memref.subview %arg0[6] [7] [-1] : memref<7xf32> to memref<7xf32, strided<[-1], offset: 6>> @@ -383,26 +383,26 @@ func.func @collapse_shape_static(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf } // CHECK-LABEL: func @collapse_shape_static // CHECK-SAME: %[[ARG:.*]]: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x3x4x1x5xf32> to !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x3x4x1x5xf32> to !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C3]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C20:.*]] = llvm.mlir.constant(20 : index) : i64 -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C20]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C20]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C4]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C4]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C5:.*]] = llvm.mlir.constant(5 : index) : i64 -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C5]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C5]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C5]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C5]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<3x4x5xf32> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<3x4x5xf32> // CHECK: return %[[RES]] : memref<3x4x5xf32> // CHECK: } @@ -418,29 +418,29 @@ func.func @collapse_shape_dynamic_with_non_identity_layout( } // CHECK-LABEL: func.func @collapse_shape_dynamic_with_non_identity_layout( // CHECK-SAME: %[[ARG:.*]]: memref<4x?x?xf32, strided<[?, 4, 1], offset: ?>>) -> memref<4x?xf32, strided<[?, ?], offset: ?>> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<4x?x?xf32, strided<[?, 4, 1], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<4x?x?xf32, strided<[?, 4, 1], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[STRIDE0_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[STRIDE0]] : i64 to index // CHECK: %[[STRIDE0:.*]] = builtin.unrealized_conversion_cast %[[STRIDE0_TO_IDX]] : index to i64 // CHECK: %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE1]], %[[SIZE2]] : i64 // CHECK: %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index // CHECK: %[[FINAL_SIZE1:.*]] = builtin.unrealized_conversion_cast %[[SIZE1_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C4]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<4x?xf32, strided<[?, ?], offset: ?>> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<4x?xf32, strided<[?, ?], offset: ?>> // CHECK: return %[[RES]] : memref<4x?xf32, strided<[?, ?], offset: ?>> // CHECK: } // CHECK32-LABEL: func @collapse_shape_dynamic_with_non_identity_layout( @@ -459,31 +459,31 @@ func.func @expand_shape_static(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32 } // CHECK-LABEL: func @expand_shape_static // CHECK-SAME: %[[ARG:.*]]: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<3x4x5xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<3x4x5xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C60:.*]] = llvm.mlir.constant(60 : index) : i64 -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C60]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C60]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C3:.*]] = llvm.mlir.constant(3 : index) : i64 -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C3]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C20:.*]] = llvm.mlir.constant(20 : index) : i64 -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C20]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C20]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C4:.*]] = llvm.mlir.constant(4 : index) : i64 -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C4]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[C4]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> // CHECK: %[[C5:.*]] = llvm.mlir.constant(5 : index) : i64 -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C5]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[C1]], %[[DESC8]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC10:.*]] = llvm.insertvalue %[[C5]], %[[DESC9]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC11:.*]] = llvm.insertvalue %[[C5]], %[[DESC10]][3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[DESC12:.*]] = llvm.insertvalue %[[C1]], %[[DESC11]][4, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC12]] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> to memref<1x3x4x1x5xf32> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C5]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[C1]], %[[DESC8]][3, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC10:.*]] = llvm.insertvalue %[[C5]], %[[DESC9]][4, 3] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC11:.*]] = llvm.insertvalue %[[C5]], %[[DESC10]][3, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[DESC12:.*]] = llvm.insertvalue %[[C1]], %[[DESC11]][4, 4] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC12]] : !llvm.struct<(ptr, ptr, i64, array<5 x i64>, array<5 x i64>)> to memref<1x3x4x1x5xf32> // CHECK: return %[[RES]] : memref<1x3x4x1x5xf32> // CHECK: } @@ -495,15 +495,15 @@ func.func @collapse_shape_fold_zero_dim(%arg0 : memref<1x1xf32>) -> memref } // CHECK-LABEL: func.func @collapse_shape_fold_zero_dim( // CHECK-SAME: %[[ARG:.*]]: memref<1x1xf32>) -> memref { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x1xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x1xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC2]] : !llvm.struct<(ptr, ptr, i64)> to memref +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC2]] : !llvm.struct<(ptr, ptr, i64)> to memref // CHECK: return %[[RES]] : memref // CHECK: } @@ -516,20 +516,20 @@ func.func @expand_shape_zero_dim(%arg0 : memref) -> memref<1x1xf32> { // CHECK-LABEL: func.func @expand_shape_zero_dim( // CHECK-SAME: %[[ARG:.*]]: memref) -> memref<1x1xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref to !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref to !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x1xf32> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x1xf32> // CHECK: return %[[RES]] : memref<1x1xf32> // CHECK: } @@ -542,13 +542,13 @@ func.func @collapse_shape_dynamic(%arg0 : memref<1x2x?xf32>) -> memref<1x?xf32> // CHECK-LABEL: func.func @collapse_shape_dynamic( // CHECK-SAME: %[[ARG:.*]]: memref<1x2x?xf32>) -> memref<1x?xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x2x?xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x2x?xf32> to !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[SIZE2:.*]] = llvm.extractvalue %[[MEM]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[FINAL_SIZE1:.*]] = llvm.mul %[[SIZE2]], %[[C2]] : i64 // CHECK: %[[SIZE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE1]] : i64 to index @@ -558,15 +558,15 @@ func.func @collapse_shape_dynamic(%arg0 : memref<1x2x?xf32>) -> memref<1x?xf32> // CHECK: %[[MIN_STRIDE1:.*]] = llvm.select %[[IS_MIN_STRIDE1]], %[[STRIDE1]], %[[C1]] : i1, i64 // CHECK: %[[MIN_STRIDE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[MIN_STRIDE1]] : i64 to index // CHECK: %[[MIN_STRIDE1:.*]] = builtin.unrealized_conversion_cast %[[MIN_STRIDE1_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[MIN_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x?xf32> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[FINAL_SIZE1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[MIN_STRIDE1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC6]] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<1x?xf32> // CHECK: return %[[RES]] : memref<1x?xf32> // CHECK: } @@ -579,12 +579,12 @@ func.func @expand_shape_dynamic(%arg0 : memref<1x?xf32>) -> memref<1x2x?xf32> { // CHECK-LABEL: func.func @expand_shape_dynamic( // CHECK-SAME: %[[ARG:.*]]: memref<1x?xf32>) -> memref<1x2x?xf32> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[CMINUS1:.*]] = llvm.mlir.constant(-1 : index) : i64 // CHECK: %[[IS_NEGATIVE_SIZE1:.*]] = llvm.icmp "slt" %[[SIZE1]], %[[C0]] : i64 @@ -595,20 +595,20 @@ func.func @expand_shape_dynamic(%arg0 : memref<1x?xf32>) -> memref<1x2x?xf32> { // CHECK: %[[FINAL_SIZE2:.*]] = llvm.select %[[IS_NEGATIVE_SIZE1]], %[[NEGATIVE_SIZE2]], %[[SIZE2]] : i1, i64 // CHECK: %[[SIZE2_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_SIZE2]] : i64 to index // CHECK: %[[FINAL_SIZE2:.*]] = builtin.unrealized_conversion_cast %[[SIZE2_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[C0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[C1]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[C2]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // In this example stride1 and size2 are the same. // Hence with CSE, we get the same SSA value. -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC6]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[C1]], %[[DESC7]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC8]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32> // CHECK: return %[[RES]] : memref<1x2x?xf32> // CHECK: } @@ -624,14 +624,14 @@ func.func @expand_shape_dynamic_with_non_identity_layout( } // CHECK-LABEL: func.func @expand_shape_dynamic_with_non_identity_layout( // CHECK-SAME: %[[ARG:.*]]: memref<1x?xf32, strided<[?, ?], offset: ?>>) -> memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> { -// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, -// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[MEM:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<1x?xf32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER:.*]] = llvm.extractvalue %[[MEM]][0] : !llvm.struct<(ptr, ptr, i64, +// CHECK: %[[ALIGNED_BUFFER:.*]] = llvm.extractvalue %[[MEM]][1] : !llvm.struct<(ptr, ptr, i64, // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[CMINUS1:.*]] = llvm.mlir.constant(-1 : index) : i64 // CHECK: %[[IS_NEGATIVE_SIZE1:.*]] = llvm.icmp "slt" %[[SIZE1]], %[[C0]] : i64 @@ -645,18 +645,18 @@ func.func @expand_shape_dynamic_with_non_identity_layout( // CHECK: %[[FINAL_STRIDE1:.*]] = llvm.mul %[[TMP_SIZE2]], %[[STRIDE1]] // CHECK: %[[STRIDE1_TO_IDX:.*]] = builtin.unrealized_conversion_cast %[[FINAL_STRIDE1]] : i64 to index // CHECK: %[[FINAL_STRIDE1:.*]] = builtin.unrealized_conversion_cast %[[STRIDE1_TO_IDX]] : index to i64 -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC1]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC2]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_BUFFER]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[ALIGNED_BUFFER]], %[[DESC1]][1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[OFFSET]], %[[DESC2]][2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 -// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC4]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C2]], %[[DESC5]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_STRIDE1]], %[[DESC6]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC7]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[STRIDE1]], %[[DESC8]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC9]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> +// CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[C1]], %[[DESC3]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[STRIDE0]], %[[DESC4]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[C2]], %[[DESC5]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC7:.*]] = llvm.insertvalue %[[FINAL_STRIDE1]], %[[DESC6]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC8:.*]] = llvm.insertvalue %[[FINAL_SIZE2]], %[[DESC7]][3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[DESC9:.*]] = llvm.insertvalue %[[STRIDE1]], %[[DESC8]][4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: %[[RES:.*]] = builtin.unrealized_conversion_cast %[[DESC9]] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> to memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> // CHECK: return %[[RES]] : memref<1x2x?xf32, strided<[?, ?, ?], offset: ?>> // CHECK: } diff --git a/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir b/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir index 5aa844c5afc5..d5ac0b0da979 100644 --- a/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/generic-functions.mlir @@ -1,13 +1,13 @@ -// RUN: mlir-opt -pass-pipeline="builtin.module(convert-memref-to-llvm{use-generic-functions=1})" -split-input-file %s \ +// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1 use-opaque-pointers=1})" -split-input-file %s \ // RUN: | FileCheck %s --check-prefix="CHECK-NOTALIGNED" -// RUN: mlir-opt -pass-pipeline="builtin.module(convert-memref-to-llvm{use-generic-functions=1 use-aligned-alloc=1})" -split-input-file %s \ +// RUN: mlir-opt -pass-pipeline="builtin.module(finalize-memref-to-llvm{use-generic-functions=1 use-aligned-alloc=1 use-opaque-pointers=1})" -split-input-file %s \ // RUN: | FileCheck %s --check-prefix="CHECK-ALIGNED" // CHECK-LABEL: func @alloc() func.func @zero_d_alloc() -> memref { -// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_alloc(%{{.*}}) : (i64) -> !llvm.ptr -// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_aligned_alloc(%{{.*}}, %{{.*}}) : (i64, i64) -> !llvm.ptr +// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_alloc(%{{.*}}) : (i64) -> !llvm.ptr +// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_aligned_alloc(%{{.*}}, %{{.*}}) : (i64, i64) -> !llvm.ptr %0 = memref.alloc() : memref return %0 : memref } @@ -16,8 +16,8 @@ func.func @zero_d_alloc() -> memref { // CHECK-LABEL: func @dealloc() func.func @dealloc(%arg0: memref) { -// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () -// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () +// CHECK-NOTALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () +// CHECK-ALIGNED: llvm.call @_mlir_memref_to_llvm_free(%{{.*}}) : (!llvm.ptr) -> () memref.dealloc %arg0 : memref return } diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir index 1a8a75d1e2b3..f6dc44cf4571 100644 --- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir +++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir @@ -1,5 +1,5 @@ -// RUN: mlir-opt -convert-memref-to-llvm %s -split-input-file | FileCheck %s -// RUN: mlir-opt -convert-memref-to-llvm='index-bitwidth=32' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=1' %s -split-input-file | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='index-bitwidth=32 use-opaque-pointers=1' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s // CHECK-LABEL: func @view( // CHECK: %[[ARG0F:.*]]: index, %[[ARG1F:.*]]: index, %[[ARG2F:.*]]: index @@ -8,80 +8,76 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) { // CHECK: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0F:.*]] // CHECK: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1F:.*]] // CHECK: llvm.mlir.constant(2048 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %0 = memref.alloc() : memref<2048xi8> // Test two dynamic sizes. - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[ARG0]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG0]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mul %{{.*}}, %[[ARG1]] - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %1 = memref.view %0[%arg2][%arg0, %arg1] : memref<2048xi8> to memref // Test one dynamic size. - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR_2:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_2]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0_2:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mul %{{.*}}, %[[ARG1]] - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %3 = memref.view %0[%arg2][%arg1] : memref<2048xi8> to memref<4x?xf32> // Test static sizes. - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR_3:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_3]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0_3:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0_3]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0_3]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(64 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %5 = memref.view %0[%arg2][] : memref<2048xi8> to memref<64x4xf32> // Test view memory space. // CHECK: llvm.mlir.constant(2048 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)> %6 = memref.alloc() : memref<2048xi8, 4> - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: %[[BASE_PTR_4:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[SHIFTED_BASE_PTR_4:.*]] = llvm.getelementptr %[[BASE_PTR_4]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: %[[CAST_SHIFTED_BASE_PTR_4:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_4]] : !llvm.ptr to !llvm.ptr - // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_4]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_4:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_4:.*]] = llvm.getelementptr %[[BASE_PTR_4]][%[[ARG2]]] : (!llvm.ptr<4>, i64) -> !llvm.ptr<4>, i8 + // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_4]], %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[C0_4:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[C0_4]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[C0_4]], %{{.*}}[2] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(64 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)> %7 = memref.view %6[%arg2][] : memref<2048xi8, 4> to memref<64x4xf32, 4> return @@ -94,17 +90,17 @@ func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) { // CHECK: %[[ARG1:.*]]: memref<0xi8>) func.func @view_empty_memref(%offset: index, %mem: memref<0xi8>) { - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: llvm.mlir.constant(4 : index) : i64 - // CHECK: = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.view %mem[%offset][] : memref<0xi8> to memref<0x4xf32> return @@ -136,10 +132,10 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in // CHECK-LABEL: func @assume_alignment func.func @assume_alignment(%0 : memref<4x4xf16>) { - // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK-NEXT: %[[ZERO:.*]] = llvm.mlir.constant(0 : index) : i64 // CHECK-NEXT: %[[MASK:.*]] = llvm.mlir.constant(15 : index) : i64 - // CHECK-NEXT: %[[INT:.*]] = llvm.ptrtoint %[[PTR]] : !llvm.ptr to i64 + // CHECK-NEXT: %[[INT:.*]] = llvm.ptrtoint %[[PTR]] : !llvm.ptr to i64 // CHECK-NEXT: %[[MASKED_PTR:.*]] = llvm.and %[[INT]], %[[MASK:.*]] : i64 // CHECK-NEXT: %[[CONDITION:.*]] = llvm.icmp "eq" %[[MASKED_PTR]], %[[ZERO]] : i64 // CHECK-NEXT: "llvm.intr.assume"(%[[CONDITION]]) : (i1) -> () @@ -159,32 +155,32 @@ func.func @dim_of_unranked(%unranked: memref<*xi32>) -> index { // CHECK: %[[UNRANKED_DESC:.*]] = builtin.unrealized_conversion_cast // CHECK: %[[RANKED_DESC:.*]] = llvm.extractvalue %[[UNRANKED_DESC]][1] -// CHECK-SAME: : !llvm.struct<(i64, ptr)> +// CHECK-SAME: : !llvm.struct<(i64, ptr)> -// CHECK: %[[ZERO_D_DESC:.*]] = llvm.bitcast %[[RANKED_DESC]] -// CHECK-SAME: : !llvm.ptr to !llvm.ptr, ptr, i64)>> - -// CHECK: %[[OFFSET_PTR:.*]] = llvm.getelementptr %[[ZERO_D_DESC]]{{\[}} -// CHECK-SAME: 0, 2] : (!llvm.ptr, ptr, -// CHECK-SAME: i64)>>) -> !llvm.ptr +// CHECK: %[[OFFSET_PTR:.*]] = llvm.getelementptr %[[RANKED_DESC]]{{\[}} +// CHECK-SAME: 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64 // CHECK: %[[INDEX_INC:.*]] = llvm.add %[[C1]], %{{.*}} : i64 // CHECK: %[[SIZE_PTR:.*]] = llvm.getelementptr %[[OFFSET_PTR]]{{\[}} -// CHECK-SAME: %[[INDEX_INC]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-SAME: %[[INDEX_INC]]] : (!llvm.ptr, i64) -> !llvm.ptr -// CHECK: %[[SIZE:.*]] = llvm.load %[[SIZE_PTR]] : !llvm.ptr +// CHECK: %[[SIZE:.*]] = llvm.load %[[SIZE_PTR]] : !llvm.ptr -> i64 -// CHECK32: %[[SIZE:.*]] = llvm.load %{{.*}} : !llvm.ptr +// CHECK32: %[[SIZE:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32 // ----- // CHECK-LABEL: func @address_space( func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) { + // CHECK: %[[MEMORY:.*]] = llvm.call @malloc(%{{.*}}) + // CHECK: %[[CAST:.*]] = llvm.addrspacecast %[[MEMORY]] : !llvm.ptr to !llvm.ptr<5> + // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[0] + // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[1] %0 = memref.alloc() : memref<32xf32, affine_map<(d0) -> (d0)>, 5> %1 = arith.constant 7 : index - // CHECK: llvm.load %{{.*}} : !llvm.ptr + // CHECK: llvm.load %{{.*}} : !llvm.ptr<5> -> f32 %2 = memref.load %0[%1] : memref<32xf32, affine_map<(d0) -> (d0)>, 5> func.return } @@ -192,16 +188,16 @@ func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) { // ----- // CHECK-LABEL: func @transpose -// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.extractvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.extractvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.extractvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> -// CHECK: llvm.insertvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.extractvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.extractvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.extractvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK: llvm.insertvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> func.func @transpose(%arg0: memref>) { %0 = memref.transpose %arg0 (i, j, k) -> (k, i, j) : memref> to memref> return @@ -227,17 +223,17 @@ func.func @get_gv0_memref() { %0 = memref.get_global @gv0 : memref<2xf32> // CHECK: %[[DIM:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[STRIDE:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv0 : !llvm.ptr> - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr>) -> !llvm.ptr + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv0 : !llvm.ptr + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x f32> // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 - // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> return } @@ -247,19 +243,19 @@ func.func @get_gv2_memref() { // CHECK: %[[DIM0:.*]] = llvm.mlir.constant(2 : index) : i64 // CHECK: %[[DIM1:.*]] = llvm.mlir.constant(3 : index) : i64 // CHECK: %[[STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv2 : !llvm.ptr>> - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0, 0] : (!llvm.ptr>>) -> !llvm.ptr + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv2 : !llvm.ptr + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x array<3 x f32>> // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 - // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DIM0]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: llvm.insertvalue %[[STRIDE1]], {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DIM0]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[STRIDE1]], {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> %0 = memref.get_global @gv2 : memref<2x3xf32> return @@ -271,15 +267,15 @@ memref.global @gv3 : memref = dense<1.0> // CHECK-LABEL: func @get_gv3_memref func.func @get_gv3_memref() { - // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv3 : !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0] : (!llvm.ptr) -> !llvm.ptr + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv3 : !llvm.ptr + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0] : (!llvm.ptr) -> !llvm.ptr, f32 // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 - // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> - // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> - // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 - // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)> %0 = memref.get_global @gv3 : memref return } @@ -321,8 +317,8 @@ func.func @rank_of_unranked(%unranked: memref<*xi32>) { return } // CHECK: %[[UNRANKED_DESC:.*]] = builtin.unrealized_conversion_cast -// CHECK-NEXT: llvm.extractvalue %[[UNRANKED_DESC]][0] : !llvm.struct<(i64, ptr)> -// CHECK32: llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i32, ptr)> +// CHECK-NEXT: llvm.extractvalue %[[UNRANKED_DESC]][0] : !llvm.struct<(i64, ptr)> +// CHECK32: llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i32, ptr)> // CHECK-LABEL: func @rank_of_ranked // CHECK32-LABEL: func @rank_of_ranked @@ -366,11 +362,11 @@ func.func @generic_atomic_rmw(%I : memref<10xi32>, %i : index) { ^bb0(%old_value : i32): memref.atomic_yield %old_value : i32 } - // CHECK: [[init:%.*]] = llvm.load %{{.*}} : !llvm.ptr + // CHECK: [[init:%.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32 // CHECK-NEXT: llvm.br ^bb1([[init]] : i32) // CHECK-NEXT: ^bb1([[loaded:%.*]]: i32): // CHECK-NEXT: [[pair:%.*]] = llvm.cmpxchg %{{.*}}, [[loaded]], [[loaded]] - // CHECK-SAME: acq_rel monotonic : i32 + // CHECK-SAME: acq_rel monotonic : !llvm.ptr, i32 // CHECK-NEXT: [[new:%.*]] = llvm.extractvalue [[pair]][0] // CHECK-NEXT: [[ok:%.*]] = llvm.extractvalue [[pair]][1] // CHECK-NEXT: llvm.cond_br [[ok]], ^bb2, ^bb1([[new]] : i32) @@ -383,28 +379,28 @@ func.func @generic_atomic_rmw(%I : memref<10xi32>, %i : index) { func.func @memref_copy_ranked() { %0 = memref.alloc() : memref<2xf32> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %1 = memref.cast %0 : memref<2xf32> to memref %2 = memref.alloc() : memref<2xf32> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %3 = memref.cast %2 : memref<2xf32> to memref memref.copy %1, %3 : memref to memref // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue {{%.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue {{%.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> // CHECK: [[MUL:%.*]] = llvm.mul [[ONE]], [[EXTRACT0]] : i64 - // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr - // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr - // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 + // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr + // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr, f32 + // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 // CHECK: [[SIZE:%.*]] = llvm.mul [[MUL]], [[PTRTOINT]] : i64 - // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr + // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 + // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK: [[VOLATILE:%.*]] = llvm.mlir.constant(false) : i1 - // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () + // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () return } @@ -420,23 +416,23 @@ func.func @memref_copy_contiguous(%in: memref<16x2xi32>, %offset: index) { // CHECK: llvm.insertvalue {{%.*}}, {{%.*}}[4, 1] // Get the memref for the subview. // CHECK: %[[SUBVIEW:.*]] = memref.subview %{{.*}}[%{{.*}}, 0] [1, 2] [1, 1] : memref<16x2xi32> to memref<1x2xi32, strided<[2, 1], offset: ?>> - // CHECK: %[[DESC:.*]] = builtin.unrealized_conversion_cast %[[SUBVIEW]] : memref<1x2xi32, strided<[2, 1], offset: ?>> to !llvm.struct<(ptr - // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue %[[DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[DESC:.*]] = builtin.unrealized_conversion_cast %[[SUBVIEW]] : memref<1x2xi32, strided<[2, 1], offset: ?>> to !llvm.struct<(ptr + // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue %[[DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: [[MUL1:%.*]] = llvm.mul {{.*}}, [[EXTRACT0]] : i64 - // CHECK: [[EXTRACT1:%.*]] = llvm.extractvalue %[[DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[EXTRACT1:%.*]] = llvm.extractvalue %[[DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> // CHECK: [[MUL2:%.*]] = llvm.mul [[MUL1]], [[EXTRACT1]] : i64 - // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr - // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr - // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 + // CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm.ptr + // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr, i32 + // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64 // CHECK: [[SIZE:%.*]] = llvm.mul [[MUL2]], [[PTRTOINT]] : i64 - // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr - // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> - // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr + // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 + // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // CHECK: [[VOLATILE:%.*]] = llvm.mlir.constant(false) : i1 - // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () + // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]], [[VOLATILE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () return } @@ -469,29 +465,28 @@ func.func @memref_copy_noncontiguous(%in: memref<16x2xi32>, %offset: index) { func.func @memref_copy_unranked() { %0 = memref.alloc() : memref<2xi1> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %1 = memref.cast %0 : memref<2xi1> to memref<*xi1> %2 = memref.alloc() : memref<2xi1> // CHECK: llvm.mlir.constant(2 : index) : i64 - // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> %3 = memref.cast %2 : memref<2xi1> to memref<*xi1> memref.copy %1, %3 : memref<*xi1> to memref<*xi1> // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[ALLOCA:%.*]] = llvm.alloca %35 x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> - // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> - // CHECK: [[BITCAST:%.*]] = llvm.bitcast [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> to !llvm.ptr + // CHECK: [[ALLOCA:%.*]] = llvm.alloca [[ONE]] x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr + // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>, !llvm.ptr // CHECK: [[RANK:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[UNDEF:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> - // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)> - // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[BITCAST]], [[INSERT]][1] : !llvm.struct<(i64, ptr)> - // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr + // CHECK: [[UNDEF:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[ALLOCA]], [[INSERT]][1] : !llvm.struct<(i64, ptr)> + // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr // CHECK: [[RANK2:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: [[ALLOCA2:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> - // CHECK: llvm.store {{%.*}}, [[ALLOCA2]] : !llvm.ptr)>> - // CHECK: [[ALLOCA3:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> - // CHECK: llvm.store [[INSERT2]], [[ALLOCA3]] : !llvm.ptr)>> + // CHECK: [[ALLOCA2:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr + // CHECK: llvm.store {{%.*}}, [[ALLOCA2]] : !llvm.struct<(i64, ptr)>, !llvm.ptr + // CHECK: [[ALLOCA3:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr + // CHECK: llvm.store [[INSERT2]], [[ALLOCA3]] : !llvm.struct<(i64, ptr)>, !llvm.ptr // CHECK: [[SIZE:%.*]] = llvm.mlir.constant(1 : index) : i64 - // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr)>>, !llvm.ptr)>>) -> () + // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr, !llvm.ptr) -> () // CHECK: llvm.intr.stackrestore [[STACKSAVE]] return } @@ -501,8 +496,8 @@ func.func @memref_copy_unranked() { // CHECK-LABEL: func @extract_aligned_pointer_as_index func.func @extract_aligned_pointer_as_index(%m: memref) -> index { %0 = memref.extract_aligned_pointer_as_index %m: memref -> index - // CHECK: %[[E:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> - // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[E]] : !llvm.ptr to i64 + // CHECK: %[[E:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[E]] : !llvm.ptr to i64 // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index // CHECK: return %[[R:.*]] : index @@ -513,19 +508,19 @@ func.func @extract_aligned_pointer_as_index(%m: memref) -> index { // CHECK-LABEL: func @extract_strided_metadata( // CHECK-SAME: %[[ARG:.*]]: memref -// CHECK: %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[ALIGNED_BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BASE]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[ALIGNED_BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BASE]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)> // CHECK: %[[OFF0:.*]] = llvm.mlir.constant(0 : index) : i64 -// CHECK: %[[BASE_BUFFER_DESC:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> -// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM_DESC]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[SIZE0:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[BASE_BUFFER_DESC:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)> +// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM_DESC]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE0:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> func.func @extract_strided_metadata( %ref: memref>) { @@ -537,3 +532,24 @@ func.func @extract_strided_metadata( return } + +// ----- + +// CHECK-LABEL: func @load_non_temporal( +func.func @load_non_temporal(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>>) { + %1 = arith.constant 7 : index + // CHECK: llvm.load %{{.*}} {nontemporal} : !llvm.ptr -> f32 + %2 = memref.load %arg0[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + func.return +} + +// ----- + +// CHECK-LABEL: func @store_non_temporal( +func.func @store_non_temporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>, %output : memref<32xf32, affine_map<(d0) -> (d0)>>) { + %1 = arith.constant 7 : index + %2 = memref.load %input[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : f32, !llvm.ptr + memref.store %2, %output[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + func.return +} diff --git a/mlir/test/Conversion/MemRefToLLVM/typed-pointers.mlir b/mlir/test/Conversion/MemRefToLLVM/typed-pointers.mlir new file mode 100644 index 000000000000..e14f70df15b9 --- /dev/null +++ b/mlir/test/Conversion/MemRefToLLVM/typed-pointers.mlir @@ -0,0 +1,415 @@ +// RUN: mlir-opt -finalize-memref-to-llvm='use-opaque-pointers=0' %s -split-input-file | FileCheck %s +// RUN: mlir-opt -finalize-memref-to-llvm='index-bitwidth=32 use-opaque-pointers=0' -split-input-file %s | FileCheck --check-prefix=CHECK32 %s + +// CHECK-LABEL: func @view( +// CHECK: %[[ARG1F:.*]]: index, %[[ARG2F:.*]]: index +func.func @view(%arg1 : index, %arg2 : index) { + // CHECK: %[[ARG2:.*]] = builtin.unrealized_conversion_cast %[[ARG2F:.*]] + // CHECK: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1F:.*]] + // CHECK: llvm.mlir.constant(2048 : index) : i64 + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + %0 = memref.alloc() : memref<2048xi8> + + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr + // CHECK: %[[CAST_SHIFTED_BASE_PTR_2:.*]] = llvm.bitcast %[[SHIFTED_BASE_PTR_2]] : !llvm.ptr to !llvm.ptr + // CHECK: llvm.insertvalue %[[CAST_SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: %[[C0_2:.*]] = llvm.mlir.constant(0 : index) : i64 + // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.constant(1 : index) : i64 + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mlir.constant(4 : index) : i64 + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + // CHECK: llvm.mul %{{.*}}, %[[ARG1]] + // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %1 = memref.view %0[%arg2][%arg1] : memref<2048xi8> to memref<4x?xf32> + return +} + +// ----- + +// CHECK: llvm.mlir.global external @gv0() {addr_space = 0 : i32} : !llvm.array<2 x f32> { +// CHECK-NEXT: %0 = llvm.mlir.undef : !llvm.array<2 x f32> +// CHECK-NEXT: llvm.return %0 : !llvm.array<2 x f32> +// CHECK-NEXT: } +memref.global @gv0 : memref<2xf32> = uninitialized + +// CHECK-LABEL: func @get_gv0_memref +func.func @get_gv0_memref() { + %0 = memref.get_global @gv0 : memref<2xf32> + // CHECK: %[[DIM:.*]] = llvm.mlir.constant(2 : index) : i64 + // CHECK: %[[STRIDE:.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv0 : !llvm.ptr> + // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr>) -> !llvm.ptr + // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64 + // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64 + // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + return +} + +// ----- + +// CHECK-LABEL: func @memref_copy_unranked +func.func @memref_copy_unranked() { + %0 = memref.alloc() : memref<2xi1> + // CHECK: llvm.mlir.constant(2 : index) : i64 + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + %1 = memref.cast %0 : memref<2xi1> to memref<*xi1> + %2 = memref.alloc() : memref<2xi1> + // CHECK: llvm.mlir.constant(2 : index) : i64 + // CHECK: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> + %3 = memref.cast %2 : memref<2xi1> to memref<*xi1> + memref.copy %1, %3 : memref<*xi1> to memref<*xi1> + // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: [[ALLOCA:%.*]] = llvm.alloca %35 x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> + // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> + // CHECK: [[BITCAST:%.*]] = llvm.bitcast [[ALLOCA]] : !llvm.ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>> to !llvm.ptr + // CHECK: [[RANK:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: [[UNDEF:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)> + // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[BITCAST]], [[INSERT]][1] : !llvm.struct<(i64, ptr)> + // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr + // CHECK: [[RANK2:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: [[ALLOCA2:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> + // CHECK: llvm.store {{%.*}}, [[ALLOCA2]] : !llvm.ptr)>> + // CHECK: [[ALLOCA3:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr)>> + // CHECK: llvm.store [[INSERT2]], [[ALLOCA3]] : !llvm.ptr)>> + // CHECK: [[SIZE:%.*]] = llvm.mlir.constant(1 : index) : i64 + // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr)>>, !llvm.ptr)>>) -> () + // CHECK: llvm.intr.stackrestore [[STACKSAVE]] + return +} + +// ----- + +// CHECK-LABEL: func @mixed_alloc( +// CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) +func.func @mixed_alloc(%arg0: index, %arg1: index) -> memref { +// CHECK-DAG: %[[M:.*]] = builtin.unrealized_conversion_cast %[[Marg]] +// CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] +// CHECK: %[[c42:.*]] = llvm.mlir.constant(42 : index) : i64 +// CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK-NEXT: %[[st0:.*]] = llvm.mul %[[N]], %[[c42]] : i64 +// CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[st0]], %[[M]] : i64 +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[c42]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[st0]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> + %0 = memref.alloc(%arg0, %arg1) : memref + return %0 : memref +} + +// ----- + +// CHECK-LABEL: func @mixed_dealloc +func.func @mixed_dealloc(%arg0: memref) { +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> +// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () + memref.dealloc %arg0 : memref + return +} + +// ----- + +// CHECK-LABEL: func @dynamic_alloc( +// CHECK: %[[Marg:.*]]: index, %[[Narg:.*]]: index) +func.func @dynamic_alloc(%arg0: index, %arg1: index) -> memref { +// CHECK-DAG: %[[M:.*]] = builtin.unrealized_conversion_cast %[[Marg]] +// CHECK-DAG: %[[N:.*]] = builtin.unrealized_conversion_cast %[[Narg]] +// CHECK-NEXT: %[[one:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK-NEXT: %[[sz:.*]] = llvm.mul %[[N]], %[[M]] : i64 +// CHECK-NEXT: %[[null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK-NEXT: %[[gep:.*]] = llvm.getelementptr %[[null]][%[[sz]]] : (!llvm.ptr, i64) -> !llvm.ptr +// CHECK-NEXT: %[[sz_bytes:.*]] = llvm.ptrtoint %[[gep]] : !llvm.ptr to i64 +// CHECK-NEXT: llvm.call @malloc(%[[sz_bytes]]) : (i64) -> !llvm.ptr +// CHECK-NEXT: llvm.bitcast %{{.*}} : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[off:.*]] = llvm.mlir.constant(0 : index) : i64 +// CHECK-NEXT: llvm.insertvalue %[[off]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[M]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[N]], %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: llvm.insertvalue %[[one]], %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> + %0 = memref.alloc(%arg0, %arg1) : memref + return %0 : memref +} + +// ----- + +// CHECK-LABEL: func @dynamic_dealloc +func.func @dynamic_dealloc(%arg0: memref) { +// CHECK: %[[ptr:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK-NEXT: %[[ptri8:.*]] = llvm.bitcast %[[ptr]] : !llvm.ptr to !llvm.ptr +// CHECK-NEXT: llvm.call @free(%[[ptri8]]) : (!llvm.ptr) -> () + memref.dealloc %arg0 : memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_static_to_dynamic +func.func @memref_cast_static_to_dynamic(%static : memref<10x42xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %static : memref<10x42xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_static_to_mixed +func.func @memref_cast_static_to_mixed(%static : memref<10x42xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %static : memref<10x42xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_dynamic_to_static +func.func @memref_cast_dynamic_to_static(%dynamic : memref) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %dynamic : memref to memref<10x12xf32> + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_dynamic_to_mixed +func.func @memref_cast_dynamic_to_mixed(%dynamic : memref) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %dynamic : memref to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_mixed_to_dynamic +func.func @memref_cast_mixed_to_dynamic(%mixed : memref<42x?xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %mixed : memref<42x?xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_mixed_to_static +func.func @memref_cast_mixed_to_static(%mixed : memref<42x?xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %mixed : memref<42x?xf32> to memref<42x1xf32> + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_mixed_to_mixed +func.func @memref_cast_mixed_to_mixed(%mixed : memref<42x?xf32>) { +// CHECK-NOT: llvm.bitcast + %0 = memref.cast %mixed : memref<42x?xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_ranked_to_unranked +// CHECK32-LABEL: func @memref_cast_ranked_to_unranked +func.func @memref_cast_ranked_to_unranked(%arg : memref<42x2x?xf32>) { +// CHECK-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)> : (i64) -> !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> +// CHECK-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> +// CHECK-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>> to !llvm.ptr +// CHECK-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i64 +// CHECK : llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i64, ptr)> +// CHECK-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i64, ptr)> +// CHECK32-DAG: %[[c:.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK32-DAG: %[[p:.*]] = llvm.alloca %[[c]] x !llvm.struct<(ptr, ptr, i32, array<3 x i32>, array<3 x i32>)> : (i64) -> !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> +// CHECK32-DAG: llvm.store %{{.*}}, %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> +// CHECK32-DAG: %[[p2:.*]] = llvm.bitcast %[[p]] : !llvm.ptr, ptr, i32, array<3 x i32>, array<3 x i32>)>> to !llvm.ptr +// CHECK32-DAG: %[[r:.*]] = llvm.mlir.constant(3 : index) : i32 +// CHECK32 : llvm.mlir.undef : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[r]], %{{.*}}[0] : !llvm.struct<(i32, ptr)> +// CHECK32-DAG: llvm.insertvalue %[[p2]], %{{.*}}[1] : !llvm.struct<(i32, ptr)> + %0 = memref.cast %arg : memref<42x2x?xf32> to memref<*xf32> + return +} + +// ----- + +// CHECK-LABEL: func @memref_cast_unranked_to_ranked +func.func @memref_cast_unranked_to_ranked(%arg : memref<*xf32>) { +// CHECK: %[[p:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)> +// CHECK-NEXT: llvm.bitcast %[[p]] : !llvm.ptr to !llvm.ptr, ptr, i64, array<4 x i64>, array<4 x i64>)>> + %0 = memref.cast %arg : memref<*xf32> to memref + return +} + +// ----- + +// CHECK-LABEL: @memref_reinterpret_cast_unranked_to_dynamic_shape +func.func @memref_reinterpret_cast_unranked_to_dynamic_shape(%offset: index, + %size_0 : index, + %size_1 : index, + %stride_0 : index, + %stride_1 : index, + %input : memref<*xf32>) { + %output = memref.reinterpret_cast %input to + offset: [%offset], sizes: [%size_0, %size_1], + strides: [%stride_0, %stride_1] + : memref<*xf32> to memref> + return +} +// CHECK-SAME: ([[OFFSETarg:%[a-z,0-9]+]]: index, +// CHECK-SAME: [[SIZE_0arg:%[a-z,0-9]+]]: index, [[SIZE_1arg:%[a-z,0-9]+]]: index, +// CHECK-SAME: [[STRIDE_0arg:%[a-z,0-9]+]]: index, [[STRIDE_1arg:%[a-z,0-9]+]]: index, +// CHECK-DAG: [[OFFSET:%.*]] = builtin.unrealized_conversion_cast [[OFFSETarg]] +// CHECK-DAG: [[SIZE_0:%.*]] = builtin.unrealized_conversion_cast [[SIZE_0arg]] +// CHECK-DAG: [[SIZE_1:%.*]] = builtin.unrealized_conversion_cast [[SIZE_1arg]] +// CHECK-DAG: [[STRIDE_0:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_0arg]] +// CHECK-DAG: [[STRIDE_1:%.*]] = builtin.unrealized_conversion_cast [[STRIDE_1arg]] +// CHECK-DAG: [[INPUT:%.*]] = builtin.unrealized_conversion_cast +// CHECK: [[OUT_0:%.*]] = llvm.mlir.undef : [[TY:!.*]] +// CHECK: [[DESCRIPTOR:%.*]] = llvm.extractvalue [[INPUT]][1] : !llvm.struct<(i64, ptr)> +// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[BASE_PTR:%.*]] = llvm.load [[BASE_PTR_PTR]] : !llvm.ptr> +// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[DESCRIPTOR]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] +// CHECK-SAME: : (!llvm.ptr>) -> !llvm.ptr> +// CHECK: [[ALIGNED_PTR:%.*]] = llvm.load [[ALIGNED_PTR_PTR]] : !llvm.ptr> +// CHECK: [[OUT_1:%.*]] = llvm.insertvalue [[BASE_PTR]], [[OUT_0]][0] : [[TY]] +// CHECK: [[OUT_2:%.*]] = llvm.insertvalue [[ALIGNED_PTR]], [[OUT_1]][1] : [[TY]] +// CHECK: [[OUT_3:%.*]] = llvm.insertvalue [[OFFSET]], [[OUT_2]][2] : [[TY]] +// CHECK: [[OUT_4:%.*]] = llvm.insertvalue [[SIZE_0]], [[OUT_3]][3, 0] : [[TY]] +// CHECK: [[OUT_5:%.*]] = llvm.insertvalue [[STRIDE_0]], [[OUT_4]][4, 0] : [[TY]] +// CHECK: [[OUT_6:%.*]] = llvm.insertvalue [[SIZE_1]], [[OUT_5]][3, 1] : [[TY]] +// CHECK: [[OUT_7:%.*]] = llvm.insertvalue [[STRIDE_1]], [[OUT_6]][4, 1] : [[TY]] + +// ----- + +// CHECK-LABEL: @memref_reshape +func.func @memref_reshape(%input : memref<2x3xf32>, %shape : memref) { + %output = memref.reshape %input(%shape) + : (memref<2x3xf32>, memref) -> memref<*xf32> + return +} +// CHECK: [[INPUT:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[INPUT_TY:!.*]] +// CHECK: [[SHAPE:%.*]] = builtin.unrealized_conversion_cast %{{.*}} to [[SHAPE_TY:!.*]] +// CHECK: [[RANK:%.*]] = llvm.extractvalue [[SHAPE]][3, 0] : [[SHAPE_TY]] +// CHECK: [[UNRANKED_OUT_O:%.*]] = llvm.mlir.undef : !llvm.struct<(i64, ptr)> +// CHECK: [[UNRANKED_OUT_1:%.*]] = llvm.insertvalue [[RANK]], [[UNRANKED_OUT_O]][0] : !llvm.struct<(i64, ptr)> + +// Compute size in bytes to allocate result ranked descriptor +// CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : index) : i64 +// CHECK: [[PTR_SIZE:%.*]] = llvm.mlir.constant(8 : index) : i64 +// CHECK: [[INDEX_SIZE:%.*]] = llvm.mlir.constant(8 : index) : i64 +// CHECK: [[DOUBLE_PTR_SIZE:%.*]] = llvm.mul [[C2]], [[PTR_SIZE]] : i64 +// CHECK: [[DESC_ALLOC_SIZE:%.*]] = llvm.add [[DOUBLE_PTR_SIZE]], %{{.*}} +// CHECK: [[UNDERLYING_DESC:%.*]] = llvm.alloca [[DESC_ALLOC_SIZE]] x i8 +// CHECK: llvm.insertvalue [[UNDERLYING_DESC]], [[UNRANKED_OUT_1]][1] + +// Set allocated, aligned pointers and offset. +// CHECK: [[ALLOC_PTR:%.*]] = llvm.extractvalue [[INPUT]][0] : [[INPUT_TY]] +// CHECK: [[ALIGN_PTR:%.*]] = llvm.extractvalue [[INPUT]][1] : [[INPUT_TY]] +// CHECK: [[OFFSET:%.*]] = llvm.extractvalue [[INPUT]][2] : [[INPUT_TY]] +// CHECK: [[BASE_PTR_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] +// CHECK-SAME: !llvm.ptr to !llvm.ptr> +// CHECK: llvm.store [[ALLOC_PTR]], [[BASE_PTR_PTR]] : !llvm.ptr> +// CHECK: [[BASE_PTR_PTR_:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[ALIGNED_PTR_PTR:%.*]] = llvm.getelementptr [[BASE_PTR_PTR_]]{{\[}}1] +// CHECK: llvm.store [[ALIGN_PTR]], [[ALIGNED_PTR_PTR]] : !llvm.ptr> +// CHECK: [[BASE_PTR_PTR__:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] : !llvm.ptr to !llvm.ptr> +// CHECK: [[OFFSET_PTR_:%.*]] = llvm.getelementptr [[BASE_PTR_PTR__]]{{\[}}2] +// CHECK: [[OFFSET_PTR:%.*]] = llvm.bitcast [[OFFSET_PTR_]] +// CHECK: llvm.store [[OFFSET]], [[OFFSET_PTR]] : !llvm.ptr + +// Iterate over shape operand in reverse order and set sizes and strides. +// CHECK: [[STRUCT_PTR:%.*]] = llvm.bitcast [[UNDERLYING_DESC]] +// CHECK-SAME: !llvm.ptr to !llvm.ptr, ptr, i64, i64)>> +// CHECK: [[SIZES_PTR:%.*]] = llvm.getelementptr [[STRUCT_PTR]]{{\[}}0, 3] +// CHECK: [[STRIDES_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[RANK]]] +// CHECK: [[SHAPE_IN_PTR:%.*]] = llvm.extractvalue [[SHAPE]][1] : [[SHAPE_TY]] +// CHECK: [[C1_:%.*]] = llvm.mlir.constant(1 : index) : i64 +// CHECK: [[RANK_MIN_1:%.*]] = llvm.sub [[RANK]], [[C1_]] : i64 +// CHECK: llvm.br ^bb1([[RANK_MIN_1]], [[C1_]] : i64, i64) + +// CHECK: ^bb1([[DIM:%.*]]: i64, [[CUR_STRIDE:%.*]]: i64): +// CHECK: [[C0_:%.*]] = llvm.mlir.constant(0 : index) : i64 +// CHECK: [[COND:%.*]] = llvm.icmp "sge" [[DIM]], [[C0_]] : i64 +// CHECK: llvm.cond_br [[COND]], ^bb2, ^bb3 + +// CHECK: ^bb2: +// CHECK: [[SIZE_PTR:%.*]] = llvm.getelementptr [[SHAPE_IN_PTR]]{{\[}}[[DIM]]] +// CHECK: [[SIZE:%.*]] = llvm.load [[SIZE_PTR]] : !llvm.ptr +// CHECK: [[TARGET_SIZE_PTR:%.*]] = llvm.getelementptr [[SIZES_PTR]]{{\[}}[[DIM]]] +// CHECK: llvm.store [[SIZE]], [[TARGET_SIZE_PTR]] : !llvm.ptr +// CHECK: [[TARGET_STRIDE_PTR:%.*]] = llvm.getelementptr [[STRIDES_PTR]]{{\[}}[[DIM]]] +// CHECK: llvm.store [[CUR_STRIDE]], [[TARGET_STRIDE_PTR]] : !llvm.ptr +// CHECK: [[UPDATE_STRIDE:%.*]] = llvm.mul [[CUR_STRIDE]], [[SIZE]] : i64 +// CHECK: [[STRIDE_COND:%.*]] = llvm.sub [[DIM]], [[C1_]] : i64 +// CHECK: llvm.br ^bb1([[STRIDE_COND]], [[UPDATE_STRIDE]] : i64, i64) + +// CHECK: ^bb3: +// CHECK: return + +// ----- + +// CHECK-LABEL: func.func @realloc_dynamic( +// CHECK-SAME: %[[arg0:.*]]: memref, +// CHECK-SAME: %[[arg1:.*]]: index) -> memref { +func.func @realloc_dynamic(%in: memref, %d: index) -> memref{ +// CHECK: %[[descriptor:.*]] = builtin.unrealized_conversion_cast %[[arg0]] +// CHECK: %[[src_dim:.*]] = llvm.extractvalue %[[descriptor]][3, 0] +// CHECK: %[[dst_dim:.*]] = builtin.unrealized_conversion_cast %[[arg1]] : index to i64 +// CHECK: %[[cond:.*]] = llvm.icmp "ugt" %[[dst_dim]], %[[src_dim]] : i64 +// CHECK: llvm.cond_br %[[cond]], ^bb1, ^bb2(%[[descriptor]] +// CHECK: ^bb1: +// CHECK: %[[dst_null:.*]] = llvm.mlir.null : !llvm.ptr +// CHECK: %[[dst_gep:.*]] = llvm.getelementptr %[[dst_null]][1] +// CHECK: %[[dst_es:.*]] = llvm.ptrtoint %[[dst_gep]] : !llvm.ptr to i64 +// CHECK: %[[dst_size:.*]] = llvm.mul %[[dst_dim]], %[[dst_es]] +// CHECK: %[[src_size:.*]] = llvm.mul %[[src_dim]], %[[dst_es]] +// CHECK: %[[new_buffer_raw:.*]] = llvm.call @malloc(%[[dst_size]]) +// CHECK: %[[new_buffer:.*]] = llvm.bitcast %[[new_buffer_raw]] : !llvm.ptr to !llvm.ptr +// CHECK: %[[old_buffer_aligned:.*]] = llvm.extractvalue %[[descriptor]][1] +// CHECK: %[[volatile:.*]] = llvm.mlir.constant(false) : i1 +// CHECK-DAG: %[[new_buffer_void:.*]] = llvm.bitcast %[[new_buffer]] : !llvm.ptr to !llvm.ptr +// CHECK-DAG: %[[old_buffer_void:.*]] = llvm.bitcast %[[old_buffer_aligned]] : !llvm.ptr to !llvm.ptr +// CHECK: "llvm.intr.memcpy"(%[[new_buffer_void]], %[[old_buffer_void]], %[[src_size]], %[[volatile]]) +// CHECK: %[[old_buffer_unaligned:.*]] = llvm.extractvalue %[[descriptor]][0] +// CHECK: %[[old_buffer_unaligned_void:.*]] = llvm.bitcast %[[old_buffer_unaligned]] : !llvm.ptr to !llvm.ptr +// CHECK: llvm.call @free(%[[old_buffer_unaligned_void]]) +// CHECK: %[[descriptor_update1:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor]][0] +// CHECK: %[[descriptor_update2:.*]] = llvm.insertvalue %[[new_buffer]], %[[descriptor_update1]][1] +// CHECK: llvm.br ^bb2(%[[descriptor_update2]] +// CHECK: ^bb2(%[[descriptor_update3:.*]]: !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>): +// CHECK: %[[descriptor_update4:.*]] = llvm.insertvalue %[[dst_dim]], %[[descriptor_update3]][3, 0] +// CHECK: %[[descriptor_update5:.*]] = builtin.unrealized_conversion_cast %[[descriptor_update4]] +// CHECK: return %[[descriptor_update5]] : memref + + %out = memref.realloc %in(%d) : memref to memref + return %out : memref +} + diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir index 4e13c5e01f2a..1c3c553f1340 100644 --- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir +++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir @@ -9,8 +9,8 @@ isOptimized = true, emissionKind = Full > -// CHECK-DAG: #[[VOID:.*]] = #llvm.di_void_result_type -#void = #llvm.di_void_result_type +// CHECK-DAG: #[[NULL:.*]] = #llvm.di_null_type +#null = #llvm.di_null_type // CHECK-DAG: #[[INT0:.*]] = #llvm.di_basic_type #int0 = #llvm.di_basic_type< @@ -30,10 +30,10 @@ sizeInBits = 64, alignInBits = 32, offsetInBits = 4 > -// CHECK-DAG: #[[PTR1:.*]] = #llvm.di_derived_type +// CHECK-DAG: #[[PTR1:.*]] = #llvm.di_derived_type #ptr1 = #llvm.di_derived_type< // Specify the name parameter. - tag = DW_TAG_pointer_type, name = "ptr1", baseType = #int0 + tag = DW_TAG_pointer_type, name = "ptr1" > // CHECK-DAG: #[[COMP0:.*]] = #llvm.di_composite_type @@ -50,15 +50,25 @@ elements = #llvm.di_subrange > -// CHECK-DAG: #[[COMP2:.*]] = #llvm.di_composite_type +// CHECK-DAG: #[[TOPLEVEL:.*]] = #llvm.di_namespace +#toplevel_namespace = #llvm.di_namespace< + name = "toplevel", exportSymbols = true +> + +// CHECK-DAG: #[[NESTED:.*]] = #llvm.di_namespace +#nested_namespace = #llvm.di_namespace< + name = "nested", scope = #toplevel_namespace, exportSymbols = false +> + +// CHECK-DAG: #[[COMP2:.*]] = #llvm.di_composite_type #comp2 = #llvm.di_composite_type< - tag = DW_TAG_class_type, name = "class_name", file = #file, scope = #file, + tag = DW_TAG_class_type, name = "class_name", file = #file, scope = #nested_namespace, flags = "TypePassByReference|NonTrivial" > -// CHECK-DAG: #[[SPTYPE0:.*]] = #llvm.di_subroutine_type +// CHECK-DAG: #[[SPTYPE0:.*]] = #llvm.di_subroutine_type #spType0 = #llvm.di_subroutine_type< - callingConvention = DW_CC_normal, types = #void, #int0, #ptr0, #ptr1, #comp0, #comp1, #comp2 + callingConvention = DW_CC_normal, types = #null, #int0, #ptr0, #ptr1, #comp0, #comp1, #comp2 > // CHECK-DAG: #[[SPTYPE1:.*]] = #llvm.di_subroutine_type diff --git a/mlir/test/Dialect/LLVMIR/func.mlir b/mlir/test/Dialect/LLVMIR/func.mlir index c4afd42e91bb..4ad8f445c784 100644 --- a/mlir/test/Dialect/LLVMIR/func.mlir +++ b/mlir/test/Dialect/LLVMIR/func.mlir @@ -89,8 +89,8 @@ module { llvm.return } - // CHECK: llvm.func @byvalattr(%{{.*}}: !llvm.ptr {llvm.byval}) - llvm.func @byvalattr(%arg0: !llvm.ptr {llvm.byval}) { + // CHECK: llvm.func @byvalattr(%{{.*}}: !llvm.ptr {llvm.byval = i32}) + llvm.func @byvalattr(%arg0: !llvm.ptr {llvm.byval = i32}) { llvm.return } @@ -267,58 +267,6 @@ module { // ----- -module { - // expected-error@+1 {{cannot attach result attributes to functions with a void return}} - llvm.func @variadic_def() -> (!llvm.void {llvm.noundef}) -} - -// ----- - -// expected-error @below{{expected llvm.align result attribute to be an integer attribute}} -llvm.func @alignattr_ret() -> (!llvm.ptr {llvm.align = 1.0 : f32}) - -// ----- - -// expected-error @below{{llvm.align attribute attached to non-pointer result}} -llvm.func @alignattr_ret() -> (i32 {llvm.align = 4}) - -// ----- - -// expected-error @below{{expected llvm.noalias result attribute to be a unit attribute}} -llvm.func @noaliasattr_ret() -> (!llvm.ptr {llvm.noalias = 1}) - -// ----- - -// expected-error @below{{llvm.noalias attribute attached to non-pointer result}} -llvm.func @noaliasattr_ret() -> (i32 {llvm.noalias}) - -// ----- - -// expected-error @below{{expected llvm.noundef result attribute to be a unit attribute}} -llvm.func @noundefattr_ret() -> (!llvm.ptr {llvm.noundef = 1}) - -// ----- - -// expected-error @below{{expected llvm.signext result attribute to be a unit attribute}} -llvm.func @signextattr_ret() -> (i32 {llvm.signext = 1}) - -// ----- - -// expected-error @below{{llvm.signext attribute attached to non-integer result}} -llvm.func @signextattr_ret() -> (f32 {llvm.signext}) - -// ----- - -// expected-error @below{{expected llvm.zeroext result attribute to be a unit attribute}} -llvm.func @zeroextattr_ret() -> (i32 {llvm.zeroext = 1}) - -// ----- - -// expected-error @below{{llvm.zeroext attribute attached to non-integer result}} -llvm.func @zeroextattr_ret() -> (f32 {llvm.zeroext}) - -// ----- - module { // expected-error@+1 {{variadic arguments must be in the end of the argument list}} llvm.func @variadic_inside(%arg0: i32, ..., %arg1: i32) diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index fe76979c9808..642639b18dee 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -33,20 +33,6 @@ llvm.func @dtor() // expected-error@+1{{'dtor' does not have a definition}} llvm.mlir.global_dtors {dtors = [@dtor], priorities = [0 : i32]} -// ----- - -// expected-error@+1{{expected llvm.noalias argument attribute to be a unit attribute}} -func.func @invalid_noalias(%arg0: i32 {llvm.noalias = 3}) { - "llvm.return"() : () -> () -} - -// ----- - -// expected-error@+1{{llvm.align argument attribute of non integer type}} -func.func @invalid_align(%arg0: i32 {llvm.align = "foo"}) { - "llvm.return"() : () -> () -} - //////////////////////////////////////////////////////////////////////////////// // Check that parser errors are properly produced and do not crash the compiler. @@ -167,6 +153,41 @@ func.func @load_non_ptr_type(%foo : f32) { // ----- +func.func @load_syncscope(%ptr : !llvm.ptr) { + // expected-error@below {{expected syncscope to be null for non-atomic access}} + %1 = "llvm.load"(%ptr) {syncscope = "singlethread"} : (!llvm.ptr) -> (f32) +} + +// ----- + +func.func @load_unsupported_ordering(%ptr : !llvm.ptr) { + // expected-error@below {{unsupported ordering 'release'}} + %1 = llvm.load %ptr atomic release {alignment = 4 : i64} : !llvm.ptr -> f32 +} + +// ----- + +func.func @load_unsupported_type(%ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'f80' for atomic access}} + %1 = llvm.load %ptr atomic monotonic {alignment = 16 : i64} : !llvm.ptr -> f80 +} + +// ----- + +func.func @load_unsupported_type(%ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'i1' for atomic access}} + %1 = llvm.load %ptr atomic monotonic {alignment = 16 : i64} : !llvm.ptr -> i1 +} + +// ----- + +func.func @load_unaligned_atomic(%ptr : !llvm.ptr) { + // expected-error@below {{expected alignment for atomic access}} + %1 = llvm.load %ptr atomic monotonic : !llvm.ptr -> f32 +} + +// ----- + func.func @store_non_llvm_type(%foo : memref, %bar : f32) { // expected-error@+1 {{expected LLVM pointer type}} llvm.store %bar, %foo : memref @@ -188,10 +209,37 @@ func.func @store_malformed_elem_type(%foo: !llvm.ptr, %bar: f32) { // ----- -func.func @call_non_function_type(%callee : !llvm.func, %arg : i8) { - // expected-error@+1 {{expected function type}} - llvm.call %callee(%arg) : !llvm.func - llvm.return +func.func @store_syncscope(%val : f32, %ptr : !llvm.ptr) { + // expected-error@below {{expected syncscope to be null for non-atomic access}} + "llvm.store"(%val, %ptr) {syncscope = "singlethread"} : (f32, !llvm.ptr) -> () +} + +// ----- + +func.func @store_unsupported_ordering(%val : f32, %ptr : !llvm.ptr) { + // expected-error@below {{unsupported ordering 'acquire'}} + llvm.store %val, %ptr atomic acquire {alignment = 4 : i64} : f32, !llvm.ptr +} + +// ----- + +func.func @store_unsupported_type(%val : f80, %ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'f80' for atomic access}} + llvm.store %val, %ptr atomic monotonic {alignment = 16 : i64} : f80, !llvm.ptr +} + +// ----- + +func.func @store_unsupported_type(%val : i1, %ptr : !llvm.ptr) { + // expected-error@below {{unsupported type 'i1' for atomic access}} + llvm.store %val, %ptr atomic monotonic {alignment = 16 : i64} : i1, !llvm.ptr +} + +// ----- + +func.func @store_unaligned_atomic(%val : f32, %ptr : !llvm.ptr) { + // expected-error@below {{expected alignment for atomic access}} + llvm.store %val, %ptr atomic monotonic : f32, !llvm.ptr } // ----- @@ -204,9 +252,43 @@ func.func @invalid_call() { // ----- -func.func @call_non_function_type(%callee : !llvm.func, %arg : i8) { - // expected-error@+1 {{expected function type}} - llvm.call %callee(%arg) : !llvm.func +func.func @call_missing_ptr_type(%callee : !llvm.func, %arg : i8) { + // expected-error@+1 {{expected indirect call to have 2 trailing types}} + llvm.call %callee(%arg) : (i8) -> (i8) + llvm.return +} + +// ----- + +func.func private @standard_func_callee() + +func.func @call_missing_ptr_type(%arg : i8) { + // expected-error@+1 {{expected direct call to have 1 trailing type}} + llvm.call @standard_func_callee(%arg) : !llvm.ptr, (i8) -> (i8) + llvm.return +} + +// ----- + +func.func @call_non_pointer_type(%callee : !llvm.func, %arg : i8) { + // expected-error@+1 {{indirect call expects a pointer as callee: '!llvm.func'}} + llvm.call %callee(%arg) : !llvm.func, (i8) -> (i8) + llvm.return +} + +// ----- + +func.func @call_non_function_type(%callee : !llvm.ptr, %arg : i8) { + // expected-error@+1 {{expected trailing function type}} + llvm.call %callee(%arg) : !llvm.ptr, !llvm.func + llvm.return +} + +// ----- + +func.func @call_void_result_type(%callee : !llvm.ptr, %arg : i8) { + // expected-error@+1 {{expected a non-void result type}} + llvm.call %callee(%arg) : !llvm.ptr, (i8) -> (!llvm.void) llvm.return } @@ -220,6 +302,14 @@ func.func @call_unknown_symbol() { // ----- +func.func @call_variadic(%callee : !llvm.ptr>, %arg : i8) { + // expected-error@+1 {{indirect calls to variadic functions are not supported}} + llvm.call %callee(%arg) : !llvm.ptr>, (i8) -> (i8) + llvm.return +} + +// ----- + func.func private @standard_func_callee() func.func @call_non_llvm() { @@ -230,7 +320,7 @@ func.func @call_non_llvm() { // ----- -func.func @call_non_llvm_indirect(%arg0 : tensor<*xi32>) { +func.func @call_non_llvm_arg(%arg0 : tensor<*xi32>) { // expected-error@+1 {{'llvm.call' op operand #0 must be LLVM dialect-compatible type}} "llvm.call"(%arg0) : (tensor<*xi32>) -> () llvm.return @@ -238,6 +328,14 @@ func.func @call_non_llvm_indirect(%arg0 : tensor<*xi32>) { // ----- +func.func @call_non_llvm_res(%callee : !llvm.ptr) { + // expected-error@+1 {{'llvm.call' op result #0 must be LLVM dialect-compatible type}} + llvm.call %callee() : !llvm.ptr, () -> (tensor<*xi32>) + llvm.return +} + +// ----- + llvm.func @callee_func(i8) -> () func.func @callee_arg_mismatch(%arg0 : i32) { @@ -274,25 +372,9 @@ func.func @indirect_callee_return_mismatch(%callee : !llvm.ptr>) { // ----- -func.func @call_too_many_results(%callee : () -> (i32,i32)) { +func.func @call_too_many_results(%callee : !llvm.ptr) { // expected-error@+1 {{expected function with 0 or 1 result}} - llvm.call %callee() : () -> (i32, i32) - llvm.return -} - -// ----- - -func.func @call_non_llvm_result(%callee : () -> (tensor<*xi32>)) { - // expected-error@+1 {{expected result to have LLVM type}} - llvm.call %callee() : () -> (tensor<*xi32>) - llvm.return -} - -// ----- - -func.func @call_non_llvm_input(%callee : (tensor<*xi32>) -> (), %arg : tensor<*xi32>) { - // expected-error@+1 {{expected LLVM types as inputs}} - llvm.call %callee(%arg) : (tensor<*xi32>) -> () + llvm.call %callee() : !llvm.ptr, () -> (i32, i32) llvm.return } @@ -591,14 +673,14 @@ func.func @atomicrmw_expected_ptr(%f32 : f32) { func.func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{expected LLVM IR element type for operand #0 to match type for operand #1}} - %0 = "llvm.atomicrmw"(%f32_ptr, %i32) {bin_op=11, ordering=1} : (!llvm.ptr, i32) -> f32 + %0 = "llvm.atomicrmw"(%f32_ptr, %i32) {bin_op=11, ordering=1} : (!llvm.ptr, i32) -> i32 llvm.return } // ----- func.func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr, %f32 : f32) { - // expected-error@+1 {{expected LLVM IR result type to match type for operand #1}} + // expected-error@+1 {{op failed to verify that result #0 and operand #1 have the same type}} %0 = "llvm.atomicrmw"(%f32_ptr, %f32) {bin_op=11, ordering=1} : (!llvm.ptr, f32) -> i32 llvm.return } @@ -607,7 +689,7 @@ func.func @atomicrmw_mismatched_operands(%f32_ptr : !llvm.ptr, %f32 : f32) func.func @atomicrmw_expected_float(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{expected LLVM IR floating point type}} - %0 = llvm.atomicrmw fadd %i32_ptr, %i32 unordered : i32 + %0 = llvm.atomicrmw fadd %i32_ptr, %i32 unordered : !llvm.ptr, i32 llvm.return } @@ -615,7 +697,7 @@ func.func @atomicrmw_expected_float(%i32_ptr : !llvm.ptr, %i32 : i32) { func.func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm.ptr, %i1 : i1) { // expected-error@+1 {{unexpected LLVM IR type for 'xchg' bin_op}} - %0 = llvm.atomicrmw xchg %i1_ptr, %i1 unordered : i1 + %0 = llvm.atomicrmw xchg %i1_ptr, %i1 unordered : !llvm.ptr, i1 llvm.return } @@ -623,7 +705,7 @@ func.func @atomicrmw_unexpected_xchg_type(%i1_ptr : !llvm.ptr, %i1 : i1) { func.func @atomicrmw_expected_int(%f32_ptr : !llvm.ptr, %f32 : f32) { // expected-error@+1 {{expected LLVM IR integer type}} - %0 = llvm.atomicrmw max %f32_ptr, %f32 unordered : f32 + %0 = llvm.atomicrmw max %f32_ptr, %f32 unordered : !llvm.ptr, f32 llvm.return } @@ -646,15 +728,24 @@ func.func @cmpxchg_mismatched_operands(%i64_ptr : !llvm.ptr, %i32 : i32) { // ----- func.func @cmpxchg_mismatched_value_operands(%ptr : !llvm.ptr, %i32 : i32, %i64 : i64) { - // expected-error@+1 {{expected both value operands to have the same type}} + // expected-error@+1 {{op failed to verify that operand #1 and operand #2 have the same type}} %0 = "llvm.cmpxchg"(%ptr, %i32, %i64) {success_ordering=2,failure_ordering=2} : (!llvm.ptr, i32, i64) -> !llvm.struct<(i32, i1)> llvm.return } + +// ----- + +func.func @cmpxchg_mismatched_result(%ptr : !llvm.ptr, %i64 : i64) { + // expected-error@+1 {{op failed to verify that result #0 has an LLVM struct type consisting of the type of operand #2 and a bool}} + %0 = "llvm.cmpxchg"(%ptr, %i64, %i64) {success_ordering=2,failure_ordering=2} : (!llvm.ptr, i64, i64) -> !llvm.struct<(i64, i64)> + llvm.return +} + // ----- func.func @cmpxchg_unexpected_type(%i1_ptr : !llvm.ptr, %i1 : i1) { // expected-error@+1 {{unexpected LLVM IR type}} - %0 = llvm.cmpxchg %i1_ptr, %i1, %i1 monotonic monotonic : i1 + %0 = llvm.cmpxchg %i1_ptr, %i1, %i1 monotonic monotonic : !llvm.ptr, i1 llvm.return } @@ -662,7 +753,7 @@ func.func @cmpxchg_unexpected_type(%i1_ptr : !llvm.ptr, %i1 : i1) { func.func @cmpxchg_at_least_monotonic_success(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{ordering must be at least 'monotonic'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 unordered monotonic : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 unordered monotonic : !llvm.ptr, i32 llvm.return } @@ -670,7 +761,7 @@ func.func @cmpxchg_at_least_monotonic_success(%i32_ptr : !llvm.ptr, %i32 : func.func @cmpxchg_at_least_monotonic_failure(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{ordering must be at least 'monotonic'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 monotonic unordered : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 monotonic unordered : !llvm.ptr, i32 llvm.return } @@ -678,7 +769,7 @@ func.func @cmpxchg_at_least_monotonic_failure(%i32_ptr : !llvm.ptr, %i32 : func.func @cmpxchg_failure_release(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{failure ordering cannot be 'release' or 'acq_rel'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel release : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel release : !llvm.ptr, i32 llvm.return } @@ -686,7 +777,7 @@ func.func @cmpxchg_failure_release(%i32_ptr : !llvm.ptr, %i32 : i32) { func.func @cmpxchg_failure_acq_rel(%i32_ptr : !llvm.ptr, %i32 : i32) { // expected-error@+1 {{failure ordering cannot be 'release' or 'acq_rel'}} - %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel acq_rel : i32 + %0 = llvm.cmpxchg %i32_ptr, %i32, %i32 acq_rel acq_rel : !llvm.ptr, i32 llvm.return } @@ -819,43 +910,10 @@ llvm.mlir.global appending @non_array_type_global_appending_linkage() : i32 // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'llvm.loop' to be a dictionary attribute}} - llvm.br ^bb4 {llvm.loop = "test"} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'parallel_access' to be an array attribute}} - llvm.br ^bb4 {llvm.loop = {parallel_access = "loop"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{expected '"loop"' to be a symbol reference}} - llvm.br ^bb4 {llvm.loop = {parallel_access = ["loop"]}} - ^bb4: - llvm.return - } -} - -// ----- - module { llvm.func @loopOptions() { // expected-error@below {{expected '@func1' to reference a metadata op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@func1]}} + llvm.br ^bb4 {loop_annotation = #llvm.loop_annotation} ^bb4: llvm.return } @@ -869,7 +927,7 @@ module { module { llvm.func @loopOptions() { // expected-error@below {{expected '@metadata' to reference an access_group op}} - llvm.br ^bb4 {llvm.loop = {parallel_access = [@metadata]}} + llvm.br ^bb4 {loop_annotation = #llvm.loop_annotation} ^bb4: llvm.return } @@ -879,49 +937,6 @@ module { // ----- -module { - llvm.func @loopOptions() { - // expected-error@below {{expected 'options' to be a `loopopts` attribute}} - llvm.br ^bb4 {llvm.loop = {options = "name"}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{unknown loop option: name}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @loopOptions() { - // expected-error@below {{loop option present twice}} - llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts}} - ^bb4: - llvm.return - } -} - -// ----- - -module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { - // expected-error@below {{attribute 'access_groups' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "access_groups" = "test" } : !llvm.ptr - llvm.return - } -} - -// ----- - module { llvm.func @accessGroups(%arg0 : !llvm.ptr) { // expected-error@below {{expected '@func1' to specify a fully qualified reference}} @@ -1019,6 +1034,28 @@ module { // ----- +module { + llvm.metadata @metadata { + llvm.access_group @group + // expected-error@below {{expected 'group' to reference a domain operation in the same region}} + llvm.alias_scope @scope { domain = @group } + } +} + +// ----- + +module { + llvm.metadata @metadata { + // expected-error@below {{expected 'domain' to reference a domain operation in the same region}} + llvm.alias_scope @scope { domain = @domain } + } + llvm.metadata @other_metadata { + llvm.alias_scope_domain @domain + } +} + +// ----- + llvm.func @wmmaLoadOp_invalid_mem_space(%arg0: !llvm.ptr, %arg1: i32) { // expected-error@+1 {{'nvvm.wmma.load' op expected source pointer in memory space 0, 1, 3}} %0 = nvvm.wmma.load %arg0, %arg1 @@ -1291,111 +1328,85 @@ llvm.mlir.global internal @side_effecting_global() : !llvm.struct<(i8)> { // ----- -// expected-error@+1 {{'llvm.struct_attrs' is permitted only in argument or result attributes}} -func.func @struct_attrs_in_op() attributes {llvm.struct_attrs = []} { - return -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_arg_type(%arg0 : i32 {llvm.struct_attrs = []}) { - return -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_pointer_arg_type(%arg0 : !llvm.ptr {llvm.struct_attrs = []}) { - return -} - -// ----- - -// expected-error@+1 {{expected 'llvm.struct_attrs' to be an array attribute}} -func.func @invalid_arg_struct_attr_value(%arg0 : !llvm.struct<(i32)> {llvm.struct_attrs = {}}) { - return +func.func @insert_vector_invalid_source_vector_size(%arg0 : vector<16385xi8>, %arg1 : vector<[16]xi8>) { + // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} + %0 = llvm.intr.vector.insert %arg0, %arg1[0] : vector<16385xi8> into vector<[16]xi8> } // ----- -// expected-error@+1 {{size of 'llvm.struct_attrs' must match the size of the annotated '!llvm.struct'}} -func.func @invalid_arg_struct_attr_size(%arg0 : !llvm.struct<(i32)> {llvm.struct_attrs = []}) { - return +func.func @insert_vector_invalid_dest_vector_size(%arg0 : vector<16xi8>, %arg1 : vector<[16385]xi8>) { + // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} + %0 = llvm.intr.vector.insert %arg0, %arg1[0] : vector<16xi8> into vector<[16385]xi8> } // ----- -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_res_type(%arg0 : i32) -> (i32 {llvm.struct_attrs = []}) { - return %arg0 : i32 +func.func @insert_scalable_into_fixed_length_vector(%arg0 : vector<[8]xf32>, %arg1 : vector<16xf32>) { + // expected-error@+1 {{op failed to verify that it is not inserting scalable into fixed-length vectors.}} + %0 = llvm.intr.vector.insert %arg0, %arg1[0] : vector<[8]xf32> into vector<16xf32> } // ----- -// expected-error@+1 {{expected 'llvm.struct_attrs' to annotate '!llvm.struct' or '!llvm.ptr>'}} -func.func @invalid_struct_attr_pointer_res_type(%arg0 : !llvm.ptr) -> (!llvm.ptr {llvm.struct_attrs = []}) { - return %arg0 : !llvm.ptr +func.func @extract_vector_invalid_source_vector_size(%arg0 : vector<[16385]xi8>) { + // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} + %0 = llvm.intr.vector.extract %arg0[0] : vector<16xi8> from vector<[16385]xi8> } // ----- -// expected-error@+1 {{expected 'llvm.struct_attrs' to be an array attribute}} -func.func @invalid_res_struct_attr_value(%arg0 : !llvm.struct<(i32)>) -> (!llvm.struct<(i32)> {llvm.struct_attrs = {}}) { - return %arg0 : !llvm.struct<(i32)> +func.func @extract_vector_invalid_result_vector_size(%arg0 : vector<[16]xi8>) { + // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} + %0 = llvm.intr.vector.extract %arg0[0] : vector<16385xi8> from vector<[16]xi8> } // ----- -// expected-error@+1 {{size of 'llvm.struct_attrs' must match the size of the annotated '!llvm.struct'}} -func.func @invalid_res_struct_attr_size(%arg0 : !llvm.struct<(i32)>) -> (!llvm.struct<(i32)> {llvm.struct_attrs = []}) { - return %arg0 : !llvm.struct<(i32)> +func.func @extract_scalable_from_fixed_length_vector(%arg0 : vector<16xf32>) { + // expected-error@+1 {{op failed to verify that it is not extracting scalable from fixed-length vectors.}} + %0 = llvm.intr.vector.extract %arg0[0] : vector<[8]xf32> from vector<16xf32> } // ----- -func.func @insert_vector_invalid_source_vector_size(%arg0 : vector<16385xi8>, %arg1 : vector<[16]xi8>) { - // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} - %0 = llvm.intr.vector.insert %arg0, %arg1[0] : vector<16385xi8> into vector<[16]xi8> +func.func @invalid_bitcast_ptr_to_i64(%arg : !llvm.ptr) { + // expected-error@+1 {{can only cast pointers from and to pointers}} + %1 = llvm.bitcast %arg : !llvm.ptr to i64 } // ----- -func.func @insert_vector_invalid_dest_vector_size(%arg0 : vector<16xi8>, %arg1 : vector<[16385]xi8>) { - // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} - %0 = llvm.intr.vector.insert %arg0, %arg1[0] : vector<16xi8> into vector<[16385]xi8> +func.func @invalid_bitcast_i64_to_ptr() { + %0 = llvm.mlir.constant(2 : i64) : i64 + // expected-error@+1 {{can only cast pointers from and to pointers}} + %1 = llvm.bitcast %0 : i64 to !llvm.ptr } // ----- -func.func @insert_scalable_into_fixed_length_vector(%arg0 : vector<[8]xf32>, %arg1 : vector<16xf32>) { - // expected-error@+1 {{op failed to verify that it is not inserting scalable into fixed-length vectors.}} - %0 = llvm.intr.vector.insert %arg0, %arg1[0] : vector<[8]xf32> into vector<16xf32> +func.func @invalid_bitcast_vec_to_ptr(%arg : !llvm.vec<4 x ptr>) { + // expected-error@+1 {{cannot cast vector of pointers to pointer}} + %0 = llvm.bitcast %arg : !llvm.vec<4 x ptr> to !llvm.ptr } // ----- -func.func @extract_vector_invalid_source_vector_size(%arg0 : vector<[16385]xi8>) { - // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} - %0 = llvm.intr.vector.extract %arg0[0] : vector<16xi8> from vector<[16385]xi8> +func.func @invalid_bitcast_ptr_to_vec(%arg : !llvm.ptr) { + // expected-error@+1 {{cannot cast pointer to vector of pointers}} + %0 = llvm.bitcast %arg : !llvm.ptr to !llvm.vec<4 x ptr> } // ----- -func.func @extract_vector_invalid_result_vector_size(%arg0 : vector<[16]xi8>) { - // expected-error@+1 {{op failed to verify that vectors are not bigger than 2^17 bits.}} - %0 = llvm.intr.vector.extract %arg0[0] : vector<16385xi8> from vector<[16]xi8> +func.func @invalid_bitcast_addr_cast(%arg : !llvm.ptr<1>) { + // expected-error@+1 {{cannot cast pointers of different address spaces, use 'llvm.addrspacecast' instead}} + %0 = llvm.bitcast %arg : !llvm.ptr<1> to !llvm.ptr } // ----- -func.func @extract_scalable_from_fixed_length_vector(%arg0 : vector<16xf32>) { - // expected-error@+1 {{op failed to verify that it is not extracting scalable from fixed-length vectors.}} - %0 = llvm.intr.vector.extract %arg0[0] : vector<[8]xf32> from vector<16xf32> +func.func @invalid_bitcast_addr_cast_vec(%arg : !llvm.vec<4 x ptr<1>>) { + // expected-error@+1 {{cannot cast pointers of different address spaces, use 'llvm.addrspacecast' instead}} + %0 = llvm.bitcast %arg : !llvm.vec<4 x ptr<1>> to !llvm.vec<4 x ptr> } - -// ----- - -#void = #llvm.di_void_result_type -// expected-error@below {{expected subroutine to have non-void argument types}} -#void_argument_type = #llvm.di_subroutine_type diff --git a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir new file mode 100644 index 000000000000..8841a1fad918 --- /dev/null +++ b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir @@ -0,0 +1,82 @@ +// RUN: mlir-opt %s | mlir-opt | FileCheck %s + +// CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +#followup = #llvm.loop_annotation + +// CHECK-DAG: #[[VECTORIZE:.*]] = #llvm.loop_vectorize +#vectorize = #llvm.loop_vectorize< + disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32, + followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[INTERLEAVE:.*]] = #llvm.loop_interleave +#interleave = #llvm.loop_interleave + +// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll +#unroll = #llvm.loop_unroll< + disable = true, count = 32 : i32, runtimeDisable = true, full = false, + followupUnrolled = #followup, followupRemainder = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[UNROLL_AND_JAM:.*]] = #llvm.loop_unroll_and_jam +#unrollAndJam = #llvm.loop_unroll_and_jam< + disable = false, count = 16 : i32, followupOuter = #followup, followupInner = #followup, + followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[LICM:.*]] = #llvm.loop_licm +#licm = #llvm.loop_licm + +// CHECK-DAG: #[[DISTRIBUTE:.*]] = #llvm.loop_distribute +#distribute = #llvm.loop_distribute< + disable = true, followupCoincident = #followup, followupSequential = #followup, + followupFallback = #followup, followupAll = #followup +> + +// CHECK-DAG: #[[PIPELINE:.*]] = #llvm.loop_pipeline +#pipeline = #llvm.loop_pipeline + +// CHECK-DAG: #[[PEELED:.*]] = #llvm.loop_peeled +#peeled = #llvm.loop_peeled + +// CHECK-DAG: #[[UNSWITCH:.*]] = #llvm.loop_unswitch +#unswitch = #llvm.loop_unswitch + +// CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation< +// CHECK-DAG: disableNonforced = false +// CHECK-DAG: mustProgress = true +// CHECK-DAG: unroll = #[[UNROLL]] +// CHECK-DAG: unrollAndJam = #[[UNROLL_AND_JAM]] +// CHECK-DAG: licm = #[[LICM]] +// CHECK-DAG: distribute = #[[DISTRIBUTE]] +// CHECK-DAG: pipeline = #[[PIPELINE]] +// CHECK-DAG: peeled = #[[PEELED]] +// CHECK-DAG: unswitch = #[[UNSWITCH]] +// CHECK-DAG: isVectorized = false +// CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2> +#loopMD = #llvm.loop_annotation + +// CHECK: llvm.func @loop_annotation +llvm.func @loop_annotation() { + // CHECK: llvm.br ^bb1 {llvm.loop = #[[LOOP_ANNOT]] + llvm.br ^bb1 {llvm.loop = #loopMD} +^bb1: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} diff --git a/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir b/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir new file mode 100644 index 000000000000..72bf45052ef1 --- /dev/null +++ b/mlir/test/Dialect/LLVMIR/parameter-attrs-invalid.mlir @@ -0,0 +1,319 @@ +// RUN: mlir-opt %s -split-input-file -verify-diagnostics + +// Argument attributes + +// expected-error@below {{"llvm.noalias" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_noalias_arg_type(%0 : i32 {llvm.noalias}) + +// ----- + +// expected-error@below {{"llvm.noalias" should be a unit attribute}} +llvm.func @invalid_noalias_attr_type(%0 : !llvm.ptr {llvm.noalias = 10 : i32}) + +// ----- + +// expected-error@below {{"llvm.readonly" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_readonly_arg_type(%0 : i32 {llvm.readonly}) + +// ----- + +// expected-error@below {{"llvm.readonly" should be a unit attribute}} +llvm.func @invalid_readonly_attr_type(%0 : i32 {llvm.readonly = i32}) + +// ----- + +// expected-error@below {{"llvm.nest" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nest_arg_type(%0 : i32 {llvm.nest}) + +// ----- + +// expected-error@below {{"llvm.nest" should be a unit attribute}} +llvm.func @invalid_nest_attr_type(%0 : i32 {llvm.nest = "foo"}) + +// ----- + +// expected-error@below {{"llvm.align" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_align_arg_type(%0 : i32 {llvm.align = 10 : i32}) + +// ----- + +// expected-error@below {{"llvm.align" should be an integer attribute}} +llvm.func @invalid_align_attr_type(%0 : i32 {llvm.align = "foo"}) + +// ----- + +// expected-error@below {{"llvm.sret" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_sret_arg_type(%0 : i32 {llvm.sret = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.sret" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_sret_attr_type(%0 : !llvm.ptr {llvm.sret = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byval" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_byval_arg_type(%0 : i32 {llvm.byval = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byval" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_byval_attr_type(%0 : !llvm.ptr> {llvm.byval = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byref" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_byref_arg_type(%0 : i32 {llvm.byref = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.byref" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_byref_attr_type(%0 : !llvm.ptr> {llvm.byref = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.inalloca" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_inalloca_arg_type(%0 : i32 {llvm.inalloca = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.inalloca" attribute attached to LLVM pointer argument of different type}} +llvm.func @invalid_inalloca_attr_type(%0 : !llvm.ptr> {llvm.inalloca = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.signext" attribute attached to non-integer LLVM type}} +llvm.func @invalid_signext_arg_type(%0 : f32 {llvm.signext}) + +// ----- + +// expected-error@below {{"llvm.signext" should be a unit attribute}} +llvm.func @invalid_signext_attr_type(%0 : i32 {llvm.signext = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.zeroext" attribute attached to non-integer LLVM type}} +llvm.func @invalid_zeroext_arg_type(%0 : f32 {llvm.zeroext}) + +// ----- + +// expected-error@below {{"llvm.zeroext" should be a unit attribute}} +llvm.func @invalid_zeroext_attr_type(%0 : i32 {llvm.zeroext = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.noundef" should be a unit attribute}} +llvm.func @invalid_noundef_attr_type(%0 : i32 {llvm.noundef = !llvm.ptr}) + +// ----- + +// expected-error@below {{"llvm.dereferenceable" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_dereferenceable_arg_type(%0 : f32 {llvm.dereferenceable = 12 : i64}) + +// ----- + +// expected-error@below {{"llvm.dereferenceable" should be an integer attribute}} +llvm.func @invalid_dereferenceable_attr_type(%0 : !llvm.ptr {llvm.dereferenceable = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.dereferenceable_or_null" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_dereferenceable_or_null_arg_type(%0 : f32 {llvm.dereferenceable_or_null = 12 : i64}) + +// ----- + +// expected-error@below {{"llvm.dereferenceable_or_null" should be an integer attribute}} +llvm.func @invalid_dereferenceable_or_null_attr_type(%0 : !llvm.ptr {llvm.dereferenceable_or_null = !llvm.struct<(i32)>}) + +// ----- + +// expected-error@below {{"llvm.inreg" should be a unit attribute}} +llvm.func @invalid_inreg_attr_type(%0 : i32 {llvm.inreg = !llvm.ptr}) + +// ----- + +// expected-error@below {{"llvm.nocapture" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nocapture_arg_type(%0 : f32 {llvm.nocapture}) + +// ----- + +// expected-error@below {{"llvm.nocapture" should be a unit attribute}} +llvm.func @invalid_nocapture_attr_type(%0 : !llvm.ptr {llvm.nocapture = f32}) + +// ----- + +// expected-error@below {{"llvm.nofree" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nofree_arg_type(%0 : f32 {llvm.nofree}) + +// ----- + +// expected-error@below {{"llvm.nofree" should be a unit attribute}} +llvm.func @invalid_nofree_attr_type(%0 : !llvm.ptr {llvm.nofree = f32}) + +// ----- + +// expected-error@below {{"llvm.nonnull" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_nonnull_arg_type(%0 : f32 {llvm.nonnull}) + +// ----- + +// expected-error@below {{"llvm.nonnull" should be a unit attribute}} +llvm.func @invalid_nonnull_attr_type(%0 : !llvm.ptr {llvm.nonnull = f32}) + +// ----- + +// expected-error@below {{"llvm.preallocated" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_preallocated_arg_type(%0 : f32 {llvm.preallocated = i64}) + +// ----- + +// expected-error@below {{"llvm.preallocated" should be a type attribute}} +llvm.func @invalid_preallocated_attr_type(%0 : !llvm.ptr {llvm.preallocated}) + +// ----- + +// expected-error@below {{"llvm.returned" should be a unit attribute}} +llvm.func @invalid_returned_attr_type(%0 : i32 {llvm.returned = !llvm.ptr}) + +// ----- + +// expected-error@below {{"llvm.alignstack" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_alignstack_arg_type(%0 : i32 {llvm.alignstack = 10 : i32}) + +// ----- + +// expected-error@below {{"llvm.alignstack" should be an integer attribute}} +llvm.func @invalid_alignstack_attr_type(%0 : i32 {llvm.alignstack = "foo"}) + +// ----- + +// expected-error@below {{"llvm.writeonly" attribute attached to non-pointer LLVM type}} +llvm.func @invalid_writeonly_arg_type(%0 : i32 {llvm.writeonly}) + +// ----- + +// expected-error@below {{"llvm.writeonly" should be a unit attribute}} +llvm.func @invalid_writeonly_attr_type(%0 : i32 {llvm.writeonly = i32}) + +// ----- + + +// Result attributes + +// expected-error@below {{cannot attach result attributes to functions with a void return}} +llvm.func @void_def() -> (!llvm.void {llvm.noundef}) + +// ----- + +// expected-error @below{{"llvm.align" should be an integer attribute}} +llvm.func @alignattr_ret() -> (!llvm.ptr {llvm.align = 1.0 : f32}) + +// ----- + +// expected-error @below{{"llvm.align" attribute attached to non-pointer LLVM type}} +llvm.func @alignattr_ret() -> (i32 {llvm.align = 4}) + +// ----- + +// expected-error @below{{"llvm.noalias" should be a unit attribute}} +llvm.func @noaliasattr_ret() -> (!llvm.ptr {llvm.noalias = 1}) + +// ----- + +// expected-error @below{{"llvm.noalias" attribute attached to non-pointer LLVM type}} +llvm.func @noaliasattr_ret() -> (i32 {llvm.noalias}) + +// ----- + +// expected-error @below{{"llvm.noundef" should be a unit attribute}} +llvm.func @noundefattr_ret() -> (!llvm.ptr {llvm.noundef = 1}) + +// ----- + +// expected-error @below{{"llvm.signext" should be a unit attribute}} +llvm.func @signextattr_ret() -> (i32 {llvm.signext = 1}) + +// ----- + +// expected-error @below{{"llvm.signext" attribute attached to non-integer LLVM type}} +llvm.func @signextattr_ret() -> (f32 {llvm.signext}) + +// ----- + +// expected-error @below{{"llvm.zeroext" should be a unit attribute}} +llvm.func @zeroextattr_ret() -> (i32 {llvm.zeroext = 1}) + +// ----- + +// expected-error @below{{"llvm.zeroext" attribute attached to non-integer LLVM type}} +llvm.func @zeroextattr_ret() -> (f32 {llvm.zeroext}) + +// ----- + +// expected-error @below{{"llvm.allocalign" is not a valid result attribute}} +llvm.func @allocalign_ret() -> (f32 {llvm.allocalign}) + +// ----- + +// expected-error @below{{"llvm.allocptr" is not a valid result attribute}} +llvm.func @allocptr_ret() -> (!llvm.ptr {llvm.allocptr}) + +// ----- + +// expected-error @below{{"llvm.byval" is not a valid result attribute}} +llvm.func @byval_ret() -> (!llvm.ptr {llvm.byval = i64}) + +// ----- + +// expected-error @below{{"llvm.byref" is not a valid result attribute}} +llvm.func @byref_ret() -> (!llvm.ptr {llvm.byref = i64}) + +// ----- + +// expected-error @below{{"llvm.inalloca" is not a valid result attribute}} +llvm.func @inalloca_ret() -> (!llvm.ptr {llvm.inalloca = i64}) + +// ----- + +// expected-error @below{{"llvm.nest" is not a valid result attribute}} +llvm.func @nest_ret() -> (!llvm.ptr {llvm.nest}) + +// ----- + +// expected-error @below{{"llvm.nocapture" is not a valid result attribute}} +llvm.func @nocapture_ret() -> (!llvm.ptr {llvm.nocapture}) + +// ----- + +// expected-error @below{{"llvm.nofree" is not a valid result attribute}} +llvm.func @nofree_ret() -> (!llvm.ptr {llvm.nofree}) + +// ----- + +// expected-error @below{{"llvm.preallocated" is not a valid result attribute}} +llvm.func @preallocated_ret() -> (!llvm.ptr {llvm.preallocated = i64}) + +// ----- + +// expected-error @below{{"llvm.readnone" is not a valid result attribute}} +llvm.func @readnone_ret() -> (!llvm.ptr {llvm.readnone}) + +// ----- + +// expected-error @below{{"llvm.readonly" is not a valid result attribute}} +llvm.func @readonly_ret() -> (!llvm.ptr {llvm.readonly}) + +// ----- + +// expected-error @below{{"llvm.alignstack" is not a valid result attribute}} +llvm.func @alignstack_ret() -> (!llvm.ptr {llvm.alignstack = 16 : i64}) + +// ----- + +// expected-error @below{{"llvm.sret" is not a valid result attribute}} +llvm.func @sret_ret() -> (!llvm.ptr {llvm.sret = i64}) + +// ----- + +// expected-error @below{{"llvm.writeonly" is not a valid result attribute}} +llvm.func @writeonly_ret() -> (!llvm.ptr {llvm.writeonly}) diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index a37b154eca54..1e49b4e3124e 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -65,14 +65,13 @@ func.func @ops(%arg0: i32, %arg1: f32, // CHECK: %[[STRUCT:.*]] = llvm.call @foo(%[[I32]]) : (i32) -> !llvm.struct<(i32, f64, i32)> // CHECK: %[[VALUE:.*]] = llvm.extractvalue %[[STRUCT]][0] : !llvm.struct<(i32, f64, i32)> // CHECK: %[[NEW_STRUCT:.*]] = llvm.insertvalue %[[VALUE]], %[[STRUCT]][2] : !llvm.struct<(i32, f64, i32)> -// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm.ptr (i32)>> -// CHECK: %{{.*}} = llvm.call %[[FUNC]](%[[I32]]) : (i32) -> !llvm.struct<(i32, f64, i32)> +// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm.ptr +// CHECK: %{{.*}} = llvm.call %[[FUNC]](%[[I32]]) : !llvm.ptr, (i32) -> !llvm.struct<(i32, f64, i32)> %17 = llvm.call @foo(%arg0) : (i32) -> !llvm.struct<(i32, f64, i32)> %18 = llvm.extractvalue %17[0] : !llvm.struct<(i32, f64, i32)> %19 = llvm.insertvalue %18, %17[2] : !llvm.struct<(i32, f64, i32)> - %20 = llvm.mlir.addressof @foo : !llvm.ptr (i32)>> - %21 = llvm.call %20(%arg0) : (i32) -> !llvm.struct<(i32, f64, i32)> - + %20 = llvm.mlir.addressof @foo : !llvm.ptr + %21 = llvm.call %20(%arg0) : !llvm.ptr, (i32) -> !llvm.struct<(i32, f64, i32)> // Terminator operations and their successors. // @@ -340,17 +339,39 @@ func.func @null() { llvm.return } +// CHECK-LABEL: @atomic_load +func.func @atomic_load(%ptr : !llvm.ptr) { + // CHECK: llvm.load %{{.*}} atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + %0 = llvm.load %ptr atomic monotonic {alignment = 4 : i64} : !llvm.ptr -> f32 + // CHECK: llvm.load volatile %{{.*}} atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr -> f32 + %1 = llvm.load volatile %ptr atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr -> f32 + llvm.return +} + +// CHECK-LABEL: @atomic_store +func.func @atomic_store(%val : f32, %ptr : !llvm.ptr) { + // CHECK: llvm.store %{{.*}}, %{{.*}} atomic monotonic {alignment = 4 : i64} : f32, !llvm.ptr + llvm.store %val, %ptr atomic monotonic {alignment = 4 : i64} : f32, !llvm.ptr + // CHECK: llvm.store volatile %{{.*}}, %{{.*}} atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : f32, !llvm.ptr + llvm.store volatile %val, %ptr atomic syncscope("singlethread") monotonic {alignment = 16 : i64} : f32, !llvm.ptr + llvm.return +} + // CHECK-LABEL: @atomicrmw -func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { - // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : f32 - %0 = llvm.atomicrmw fadd %ptr, %val monotonic : f32 +func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) { + // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32 + %0 = llvm.atomicrmw fadd %ptr, %val monotonic : !llvm.ptr, f32 + // CHECK: llvm.atomicrmw volatile fsub %{{.*}}, %{{.*}} syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32 + %1 = llvm.atomicrmw volatile fsub %ptr, %val syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32 llvm.return } // CHECK-LABEL: @cmpxchg -func.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %new : i32) { - // CHECK: llvm.cmpxchg %{{.*}}, %{{.*}}, %{{.*}} acq_rel monotonic : i32 - %0 = llvm.cmpxchg %ptr, %cmp, %new acq_rel monotonic : i32 +func.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %new : i32) { + // CHECK: llvm.cmpxchg %{{.*}}, %{{.*}}, %{{.*}} acq_rel monotonic : !llvm.ptr, i32 + %0 = llvm.cmpxchg %ptr, %cmp, %new acq_rel monotonic : !llvm.ptr, i32 + // CHECK: llvm.cmpxchg weak volatile %{{.*}}, %{{.*}}, %{{.*}} syncscope("singlethread") acq_rel monotonic {alignment = 16 : i64} : !llvm.ptr, i32 + %1 = llvm.cmpxchg weak volatile %ptr, %cmp, %new syncscope("singlethread") acq_rel monotonic {alignment = 16 : i64} : !llvm.ptr, i32 llvm.return } @@ -401,8 +422,15 @@ llvm.func @invokeLandingpad() -> i32 attributes { personality = @__gxx_personali llvm.invoke @bar(%8, %6, %4) to ^bb2 unwind ^bb1 : (!llvm.ptr, !llvm.ptr, !llvm.ptr) -> () // CHECK: ^[[BB4:.*]]: -// CHECK: llvm.return %[[a0]] : i32 +// CHECK: %[[FUNC:.*]] = llvm.mlir.addressof @foo : !llvm.ptr +// CHECK: %{{.*}} = llvm.invoke %[[FUNC]]{{.*}}: !llvm.ptr, ^bb4: + %12 = llvm.mlir.addressof @foo : !llvm.ptr + %13 = llvm.invoke %12(%7) to ^bb2 unwind ^bb1 : !llvm.ptr, (i32) -> !llvm.struct<(i32, f64, i32)> + +// CHECK: ^[[BB5:.*]]: +// CHECK: llvm.return %[[a0]] : i32 +^bb5: llvm.return %0 : i32 } @@ -491,22 +519,6 @@ func.func @fastmathFlags(%arg0: f32, %arg1: f32, %arg2: i32, %arg3: vector<2 x f return } -module { - // CHECK-LABEL: @loopOptions - llvm.func @loopOptions() { - // CHECK: llvm.br - // CHECK-SAME: llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - llvm.br ^bb1 {llvm.loop = {options = #llvm.loopopts}, parallel_access = [@metadata::@group1]} - ^bb1: - llvm.return - } - // CHECK: llvm.metadata @metadata attributes {test_attribute} { - llvm.metadata @metadata attributes {test_attribute} { - // CHECK: llvm.access_group @group1 - llvm.access_group @group1 - } -} - // CHECK-LABEL: llvm.func @vararg_func llvm.func @vararg_func(%arg0: i32, ...) { // CHECK: %{{.*}} = llvm.mlir.constant(1 : i32) : i32 diff --git a/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir b/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir index 0513dc77ff86..a747d596bd96 100644 --- a/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/tbaa-invalid.mlir @@ -8,7 +8,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@tbaa_tag_1' to specify a fully qualified reference}} - llvm.store %0, %arg0 {llvm.tbaa = [@tbaa_tag_1]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@tbaa_tag_1]} : i8, !llvm.ptr llvm.return } } @@ -17,8 +17,8 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 - // expected-error@below {{attribute 'llvm.tbaa' failed to satisfy constraint: symbol ref array attribute}} - llvm.store %0, %arg0 {llvm.tbaa = ["sym"]} : i8, !llvm.ptr + // expected-error@below {{attribute 'tbaa' failed to satisfy constraint: symbol ref array attribute}} + llvm.store %0, %arg0 {tbaa = ["sym"]} : i8, !llvm.ptr llvm.return } @@ -28,7 +28,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@metadata::@group1' to resolve to a llvm.tbaa_tag}} - llvm.store %0, %arg0 {llvm.tbaa = [@metadata::@group1]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@metadata::@group1]} : i8, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -42,7 +42,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@metadata::@sym' to be a valid reference}} - llvm.store %0, %arg0 {llvm.tbaa = [@metadata::@sym]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@metadata::@sym]} : i8, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -54,7 +54,7 @@ module { llvm.func @tbaa(%arg0: !llvm.ptr) { %0 = llvm.mlir.constant(1 : i8) : i8 // expected-error@below {{expected '@tbaa::@sym' to reference a metadata op}} - llvm.store %0, %arg0 {llvm.tbaa = [@tbaa::@sym]} : i8, !llvm.ptr + llvm.store %0, %arg0 {tbaa = [@tbaa::@sym]} : i8, !llvm.ptr llvm.return } diff --git a/mlir/test/Dialect/MemRef/canonicalize.mlir b/mlir/test/Dialect/MemRef/canonicalize.mlir index 3d9f71e26055..14c570759b4c 100644 --- a/mlir/test/Dialect/MemRef/canonicalize.mlir +++ b/mlir/test/Dialect/MemRef/canonicalize.mlir @@ -894,3 +894,15 @@ func.func @fold_trivial_subviews(%m: memref>, to memref> return %1 : memref> } + +// ----- + +// CHECK-LABEL: func @load_store_nontemporal( +func.func @load_store_nontemporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>, %output : memref<32xf32, affine_map<(d0) -> (d0)>>) { + %1 = arith.constant 7 : index + // CHECK: memref.load %{{.*}}[%{{.*}}] {nontemporal = true} : memref<32xf32> + %2 = memref.load %input[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + // CHECK: memref.store %{{.*}}, %{{.*}}[%{{.*}}] {nontemporal = true} : memref<32xf32> + memref.store %2, %output[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>> + func.return +} diff --git a/mlir/test/Dialect/MemRef/emulate-wide-int.mlir b/mlir/test/Dialect/MemRef/emulate-wide-int.mlir index de1cba5c0477..65ac5beed0a1 100644 --- a/mlir/test/Dialect/MemRef/emulate-wide-int.mlir +++ b/mlir/test/Dialect/MemRef/emulate-wide-int.mlir @@ -44,3 +44,19 @@ func.func @alloc_load_store_i64() { memref.store %c1, %m[%c0] : memref<4xi64, 1> return } + + +// CHECK-LABEL: func @alloc_load_store_i64_nontemporal +// CHECK: [[C1:%.+]] = arith.constant dense<[1, 0]> : vector<2xi32> +// CHECK-NEXT: [[M:%.+]] = memref.alloc() : memref<4xvector<2xi32>, 1> +// CHECK-NEXT: [[V:%.+]] = memref.load [[M]][{{%.+}}] {nontemporal = true} : memref<4xvector<2xi32>, 1> +// CHECK-NEXT: memref.store [[C1]], [[M]][{{%.+}}] {nontemporal = true} : memref<4xvector<2xi32>, 1> +// CHECK-NEXT: return +func.func @alloc_load_store_i64_nontemporal() { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : i64 + %m = memref.alloc() : memref<4xi64, 1> + %v = memref.load %m[%c0] {nontemporal = true} : memref<4xi64, 1> + memref.store %c1, %m[%c0] {nontemporal = true} : memref<4xi64, 1> + return +} diff --git a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir index c2ecc90be8dd..f0d5008f991a 100644 --- a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir +++ b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir @@ -502,3 +502,25 @@ func.func @subview_of_subview_rank_reducing(%m: memref, to memref> return %1 : memref> } + +// ----- + +// CHECK-LABEL: func @fold_load_keep_nontemporal( +// CHECK: memref.load %{{.+}}[%{{.+}}, %{{.+}}] {nontemporal = true} +func.func @fold_load_keep_nontemporal(%arg0 : memref<12x32xf32>, %arg1 : index, %arg2 : index, %arg3 : index, %arg4 : index) -> f32 { + %0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] : memref<12x32xf32> to memref<4x4xf32, strided<[64, 3], offset: ?>> + %1 = memref.load %0[%arg3, %arg4] {nontemporal = true }: memref<4x4xf32, strided<[64, 3], offset: ?>> + return %1 : f32 +} + + +// ----- + +// CHECK-LABEL: func @fold_store_keep_nontemporal( +// CHECK: memref.store %{{.+}}, %{{.+}}[%{{.+}}, %{{.+}}] {nontemporal = true} : memref<12x32xf32> +func.func @fold_store_keep_nontemporal(%arg0 : memref<12x32xf32>, %arg1 : index, %arg2 : index, %arg3 : index, %arg4 : index, %arg5 : f32) { + %0 = memref.subview %arg0[%arg1, %arg2][4, 4][2, 3] : + memref<12x32xf32> to memref<4x4xf32, strided<[64, 3], offset: ?>> + memref.store %arg5, %0[%arg3, %arg4] {nontemporal=true}: memref<4x4xf32, strided<[64, 3], offset: ?>> + return +} diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir index 9d6999db12ab..1bdd5981f656 100644 --- a/mlir/test/Dialect/OpenMP/invalid.mlir +++ b/mlir/test/Dialect/OpenMP/invalid.mlir @@ -524,7 +524,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -1288,7 +1288,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw add %arg2, %2 monotonic : i32 + llvm.atomicrmw add %arg2, %2 monotonic : !llvm.ptr, i32 omp.yield } diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir index 56d898e20c3b..ac8fea8ca8f0 100644 --- a/mlir/test/Dialect/OpenMP/ops.mlir +++ b/mlir/test/Dialect/OpenMP/ops.mlir @@ -556,7 +556,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } diff --git a/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir b/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir index 23853acfa898..53f19b5f95dd 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/microbench-linalg-async-parallel-for.mlir @@ -9,7 +9,7 @@ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -23,7 +23,7 @@ // RUN: -convert-linalg-to-loops \ // RUN: -convert-scf-to-cf \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir b/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir index 046e419d9e1d..74531befff18 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/microbench-scf-async-parallel-for.mlir @@ -9,7 +9,7 @@ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -30,7 +30,7 @@ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -44,7 +44,7 @@ // RUN: -convert-linalg-to-loops \ // RUN: -convert-scf-to-cf \ // RUN: -convert-vector-to-llvm \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir index 27e143388363..61a92aa1d790 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-1d.mlir @@ -4,7 +4,7 @@ // RUN: -async-runtime-ref-counting-opt \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-func-to-llvm \ @@ -20,7 +20,7 @@ // RUN: -async-runtime-policy-based-ref-counting \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-func-to-llvm \ @@ -39,7 +39,7 @@ // RUN: -async-runtime-ref-counting-opt \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -arith-expand \ // RUN: -memref-expand \ // RUN: -convert-func-to-llvm \ diff --git a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir index 4e089135aa8b..46da16985ff6 100644 --- a/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir +++ b/mlir/test/Integration/Dialect/Async/CPU/test-async-parallel-for-2d.mlir @@ -5,7 +5,7 @@ // RUN: -arith-expand \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -20,7 +20,7 @@ // RUN: -arith-expand \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ @@ -38,7 +38,7 @@ // RUN: -arith-expand \ // RUN: -convert-async-to-llvm \ // RUN: -convert-scf-to-cf \ -// RUN: -convert-memref-to-llvm \ +// RUN: -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts \ // RUN: | mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir b/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir index c89a146bd0a7..dfefb0b14918 100644 --- a/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir +++ b/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s \ // RUN: -func-bufferize -tensor-bufferize -arith-bufferize --canonicalize \ // RUN: -convert-scf-to-cf --convert-complex-to-standard \ -// RUN: -convert-memref-to-llvm -convert-math-to-llvm -convert-math-to-libm \ +// RUN: -finalize-memref-to-llvm -convert-math-to-llvm -convert-math-to-libm \ // RUN: -convert-vector-to-llvm -convert-complex-to-llvm \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts |\ // RUN: mlir-cpu-runner \ diff --git a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir index a42c64d57a87..470614e97325 100644 --- a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir +++ b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-complex-sparse-constant.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --convert-memref-to-llvm | \ +// RUN: mlir-opt %s --finalize-memref-to-llvm | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void // diff --git a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir index e8e5ef392762..e39d97ae2933 100644 --- a/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir +++ b/mlir/test/Integration/Dialect/LLVMIR/CPU/test-vp-intrinsic.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-llvm -convert-memref-to-llvm \ +// RUN: mlir-opt %s -convert-vector-to-llvm -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry \ diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir index e801a85c87d3..ae37b2195f6a 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/matmul-vs-matvec.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -O3 -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir index f6617f145bc1..351394ad3188 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/rank-reducing-subview.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -O3 -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir index c4714089c2f1..b0529c8b97ad 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-collapse-tensor.mlir @@ -2,7 +2,7 @@ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation -convert-linalg-to-llvm \ // RUN: -expand-strided-metadata -lower-affine \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir index 6e66dec6a9fe..cb09816c1efd 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir index cbb55895b1ac..315914bf29d5 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-1d-nwc-wcf-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir index ca5d0c9213d3..dbdfd35864f1 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir index 410de4bbd575..0b009d0540a6 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-2d-nhwc-hwcf-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir index a5a89a5d7606..9e510ffd36c3 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir index 4a3e35659d4a..f8372fb5b474 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-conv-3d-ndhwc-dhwcf-call.mlir @@ -1,10 +1,10 @@ -// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s // RUN: mlir-opt %s -test-transform-dialect-interpreter -test-transform-dialect-erase-schedule -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir index 9e14c971049c..de0babae03bc 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-elementwise.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s -convert-elementwise-to-linalg \ // RUN: -arith-bufferize -linalg-bufferize -tensor-bufferize \ // RUN: -func-bufferize -buffer-deallocation -convert-linalg-to-loops \ -// RUN: -convert-linalg-to-llvm --convert-memref-to-llvm -convert-func-to-llvm \ +// RUN: -convert-linalg-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm \ // RUN: -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir index 868d940d233d..819589be27b5 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-expand-tensor.mlir @@ -2,7 +2,7 @@ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation -convert-linalg-to-llvm \ // RUN: -expand-strided-metadata -lower-affine \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir index d41a96428306..40a5cc452e2a 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-one-shot-bufferize.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(canonicalize,cse),one-shot-bufferize{bufferize-function-boundaries})" |\ // RUN: mlir-opt -pass-pipeline="builtin.module(func.func(buffer-deallocation,convert-vector-to-scf,lower-affine,convert-linalg-to-loops))" |\ -// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(canonicalize,convert-scf-to-cf),convert-vector-to-llvm,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(canonicalize,convert-scf-to-cf),convert-vector-to-llvm,expand-strided-metadata,lower-affine,convert-arith-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -O3 -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext |\ diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir index 6a255b8cd8a4..fd035881d008 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-padtensor.mlir @@ -2,7 +2,7 @@ // RUN: -empty-tensor-to-alloc-tensor -linalg-bufferize -arith-bufferize \ // RUN: -bufferization-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation \ -// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir index 2f3d971c3df7..9f23f55bf66b 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert-multiple-uses.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s -linalg-bufferize \ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation \ -// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir index 0fc72ba6618d..4d05ede756d6 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-subtensor-insert.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt %s -linalg-bufferize \ // RUN: -arith-bufferize -tensor-bufferize -func-bufferize \ // RUN: -finalizing-bufferize -buffer-deallocation \ -// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-loops -convert-scf-to-cf -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir index 2c9718cc1e26..b371421262cc 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-e2e.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -arith-bufferize -linalg-bufferize \ // RUN: -tensor-bufferize -func-bufferize -finalizing-bufferize -buffer-deallocation -convert-linalg-to-loops \ -// RUN: -convert-linalg-to-llvm --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir index 80fb41b96d11..af51df42eb4b 100644 --- a/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir +++ b/mlir/test/Integration/Dialect/Linalg/CPU/test-tensor-matmul.mlir @@ -1,7 +1,7 @@ // UNSUPPORTED: asan // RUN: mlir-opt %s -test-transform-dialect-erase-schedule -linalg-bufferize -arith-bufferize \ // RUN: -tensor-bufferize -func-bufferize -finalizing-bufferize -buffer-deallocation -convert-linalg-to-loops -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s @@ -10,7 +10,7 @@ // RUN: -scf-bufferize -arith-bufferize -tensor-bufferize \ // RUN: -func-bufferize \ // RUN: -finalizing-bufferize -convert-linalg-to-loops -convert-scf-to-cf -convert-scf-to-cf \ -// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -convert-linalg-to-llvm -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-scf-to-cf --finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext,%mlir_lib_dir/libmlir_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir b/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir index 7da0b00337f2..9221d8eb3586 100644 --- a/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir +++ b/mlir/test/Integration/Dialect/Memref/cast-runtime-verification.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -generate-runtime-verification -convert-memref-to-llvm \ +// RUN: mlir-opt %s -generate-runtime-verification -finalize-memref-to-llvm \ // RUN: -test-cf-assert \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ diff --git a/mlir/test/Integration/Dialect/Memref/memref_abi.c b/mlir/test/Integration/Dialect/Memref/memref_abi.c index 12079171f549..8862727a539a 100644 --- a/mlir/test/Integration/Dialect/Memref/memref_abi.c +++ b/mlir/test/Integration/Dialect/Memref/memref_abi.c @@ -3,7 +3,7 @@ // Compile the MLIR file to LLVM: // RUN: mlir-opt %t/input.mlir \ -// RUN: -lower-affine -convert-scf-to-cf -convert-memref-to-llvm \ +// RUN: -lower-affine -convert-scf-to-cf -finalize-memref-to-llvm \ // RUN: -convert-func-to-llvm -reconcile-unrealized-casts \ // RUN: | mlir-translate --mlir-to-llvmir -o %t.ll diff --git a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir index 5372280e8a01..d289be9799cf 100644 --- a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir +++ b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf,memref-expand,arith-expand),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf,memref-expand,arith-expand),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir b/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir index 55a98a8958cb..50cac3269891 100644 --- a/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir +++ b/mlir/test/Integration/Dialect/Standard/CPU/test_subview.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -expand-strided-metadata -lower-affine -convert-arith-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -expand-strided-metadata -lower-affine -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir index b0d22c5cc986..0e8440e4cfaf 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf-full.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf \ // RUN: -arith-bufferize -convert-vector-to-llvm="enable-amx" \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" \ // RUN: --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir index f346fc203930..ff77efea416c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir index dabed01de2b2..086d6af1049f 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-ext.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir index 14c79b7387b5..b2c51234249c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli-full.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf \ // RUN: -arith-bufferize -convert-vector-to-llvm="enable-amx" \ -// RUN: -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" \ // RUN: --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir index 6c86372062f5..ec431d4b9bf2 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir index 10e34e0d0edf..0b108fbb12b1 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir index 0c8f086ebabb..418c3e86097c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-amx" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir index a0688dcf16d9..794ed761e65b 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-arm-sve" -convert-memref-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -canonicalize | \ +// RUN: mlir-opt %s -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-arm-sve" -finalize-memref-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -canonicalize | \ // RUN: mlir-translate -mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --march=aarch64 --mattr="+sve" --dlopen=%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir index 30a2e40aadc2..6295dfaa4ec3 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-inline-asm-vector-avx512.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-linalg-to-loops -convert-vector-to-scf='full-unroll=true' -lower-affine -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm='use-bare-ptr-memref-call-conv=1' -convert-arith-to-llvm -reconcile-unrealized-casts |\ +// RUN: mlir-opt %s -convert-linalg-to-loops -convert-vector-to-scf='full-unroll=true' -lower-affine -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm='use-bare-ptr-memref-call-conv=1' -convert-arith-to-llvm -reconcile-unrealized-casts |\ // RUN: mlir-translate --mlir-to-llvmir |\ // RUN: %lli --entry-function=entry --mattr="avx512f" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext |\ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir index abfc871925bb..cde451cb48e9 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/X86Vector/test-sparse-dot-product.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm="enable-x86vector" -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm="enable-x86vector" -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-translate --mlir-to-llvmir | \ // RUN: %lli --entry-function=entry --mattr="avx512bw,avx512vp2intersect" --dlopen=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir index 1e3f65354e89..7fc4e4b95dd1 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-0-d-vectors.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir index 4ef83ddf8f02..60f653d0f8d4 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-compress.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir index b7507025e2c7..381a3d6719c7 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-expand.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir index 0b177497bb4f..5d9d4d88d2e0 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-gather.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir index b7a0de72a5c9..160c2a4ebecd 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedload.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir index 93b355aca36c..717300f785bd 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-maskedstore.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir index 518b1554505f..62975a93e452 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-realloc.mlir @@ -1,7 +1,7 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts |\ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts |\ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm='use-aligned-alloc=1' -convert-func-to-llvm -arith-expand -reconcile-unrealized-casts |\ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm='use-aligned-alloc=1' -convert-func-to-llvm -arith-expand -reconcile-unrealized-casts |\ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir index 1358840bf2e6..34f8bbd61230 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-scatter.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir index 29aab234014f..6c8a24d54b8a 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-dot-matvec.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir index 457ddb33214f..2ff69ad8f72c 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-sparse-saxpy-jagged-matvec.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir index 7e622da7aa79..c3df2665ca3f 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-1d.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},expand-strided-metadata,lower-affine,convert-arith-to-llvm,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir index 2f38ef674dd3..60020c0993fd 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-2d.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir index a49349b0f49c..f60303b9cdf2 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read-3d.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir index ccafa5b36a7c..27c22f5070f4 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-read.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir index 929a30a4d478..2e25c9fee3cb 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-to-loops.mlir @@ -1,9 +1,9 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true},lower-affine,convert-scf-to-cf),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir index 4dbdc69fdb4d..9bf3b46fbc33 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-transfer-write.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm -convert-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ +// RUN: mlir-opt %s -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | \ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \ // RUN: FileCheck %s diff --git a/mlir/test/Target/LLVMIR/Import/basic.ll b/mlir/test/Target/LLVMIR/Import/basic.ll index 050197c90c83..1ce3dedcc03a 100644 --- a/mlir/test/Target/LLVMIR/Import/basic.ll +++ b/mlir/test/Target/LLVMIR/Import/basic.ll @@ -1,5 +1,5 @@ -; RUN: mlir-translate -opaque-pointers=0 -import-llvm %s | FileCheck %s -; RUN: mlir-translate -opaque-pointers=0 -import-llvm -mlir-print-debuginfo %s | FileCheck %s --check-prefix=CHECK-DBG +; RUN: mlir-translate -import-llvm %s | FileCheck %s +; RUN: mlir-translate -import-llvm -mlir-print-debuginfo %s | FileCheck %s --check-prefix=CHECK-DBG ; CHECK-DBG: #[[MODULELOC:.+]] = loc({{.*}}basic.ll{{.*}}:0:0) @@ -16,16 +16,16 @@ declare float @fe(i32) ; CHECK: %[[c42:[0-9]+]] = llvm.mlir.constant(42 : i32) : i32 define internal dso_local i32 @f1(i64 %a) norecurse { entry: -; CHECK: %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr - %aa = inttoptr i64 %a to i64* -; CHECK-DBG: llvm.mlir.addressof @global : !llvm.ptr loc(#[[MODULELOC]]) -; %[[addrof:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr -; %[[addrof2:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr -; %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr -; %{{[0-9]+}} = llvm.ptrtoint %[[addrof2]] : !llvm.ptr to i64 -; %{{[0-9]+}} = llvm.getelementptr %[[addrof]][%3] : (!llvm.ptr, i32) -> !llvm.ptr - %bb = ptrtoint double* @global to i64 - %cc = getelementptr double, double* @global, i32 3 +; CHECK: %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr + %aa = inttoptr i64 %a to ptr +; CHECK-DBG: llvm.mlir.addressof @global : !llvm.ptr loc(#[[MODULELOC]]) +; %[[addrof:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr +; %[[addrof2:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr +; %{{[0-9]+}} = llvm.inttoptr %arg0 : i64 to !llvm.ptr +; %{{[0-9]+}} = llvm.ptrtoint %[[addrof2]] : !llvm.ptr to i64 +; %{{[0-9]+}} = llvm.getelementptr %[[addrof]][%3] : (!llvm.ptr, i32) -> !llvm.ptr + %bb = ptrtoint ptr @global to i64 + %cc = getelementptr double, ptr @global, i32 3 ; CHECK: %[[b:[0-9]+]] = llvm.trunc %arg0 : i64 to i32 ; CHECK-DBG: llvm.trunc %arg0 : i64 to i32 loc(#[[MODULELOC]]) %b = trunc i64 %a to i32 @@ -54,35 +54,33 @@ if.end: ; CHECK-DBG: } loc(#[[MODULELOC]]) -@_ZTIi = external dso_local constant i8* -@_ZTIii= external dso_local constant i8** -declare void @foo(i8*) -declare i8* @bar(i8*) +@_ZTIi = external dso_local constant ptr +@_ZTIii= external dso_local constant ptr +declare void @foo(ptr) +declare ptr @bar(ptr) declare i32 @__gxx_personality_v0(...) ; CHECK-LABEL: @invokeLandingpad -define i32 @invokeLandingpad() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { - ; CHECK: %[[a1:[0-9]+]] = llvm.bitcast %{{[0-9]+}} : !llvm.ptr>> to !llvm.ptr - ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x i8 {alignment = 1 : i64} : (i32) -> !llvm.ptr +define i32 @invokeLandingpad() personality ptr @__gxx_personality_v0 { + ; CHECK: %[[a1:[0-9]+]] = llvm.mlir.addressof @_ZTIii : !llvm.ptr + ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x i8 {alignment = 1 : i64} : (i32) -> !llvm.ptr %1 = alloca i8 - ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> () - invoke void @foo(i8* %1) to label %4 unwind label %2 + ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> () + invoke void @foo(ptr %1) to label %4 unwind label %2 ; CHECK: ^bb1: - ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr>) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i8>) : !llvm.struct<(ptr, i32)> - %3 = landingpad { i8*, i32 } catch i8** @_ZTIi catch i8* bitcast (i8*** @_ZTIii to i8*) - ; FIXME: Change filter to a constant array once they are handled. - ; Currently, even though it parses this, LLVM module is broken - filter [1 x i8] [i8 1] - resume { i8*, i32 } %3 + ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i1>) : !llvm.struct<(ptr, i32)> + %3 = landingpad { ptr, i32 } catch ptr @_ZTIi catch ptr @_ZTIii + filter [1 x i1] [i1 1] + resume { ptr, i32 } %3 ; CHECK: ^bb2: ; CHECK: llvm.return %{{[0-9]+}} : i32 ret i32 1 ; CHECK: ^bb3: - ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> !llvm.ptr - %6 = invoke i8* @bar(i8* %1) to label %4 unwind label %2 + ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> !llvm.ptr + %6 = invoke ptr @bar(ptr %1) to label %4 unwind label %2 ; CHECK: ^bb4: ; CHECK: llvm.return %{{[0-9]+}} : i32 @@ -107,32 +105,28 @@ define i32 @useFreezeOp(i32 %x) { } ; Varadic function definition -%struct.va_list = type { i8* } +%struct.va_list = type { ptr } -declare void @llvm.va_start(i8*) -declare void @llvm.va_copy(i8*, i8*) -declare void @llvm.va_end(i8*) +declare void @llvm.va_start(ptr) +declare void @llvm.va_copy(ptr, ptr) +declare void @llvm.va_end(ptr) ; CHECK-LABEL: llvm.func @variadic_function define void @variadic_function(i32 %X, ...) { - ; CHECK: %[[ALLOCA0:.+]] = llvm.alloca %{{.*}} x !llvm.struct<"struct.va_list", (ptr)> {{.*}} : (i32) -> !llvm.ptr)>> + ; CHECK: %[[ALLOCA0:.+]] = llvm.alloca %{{.*}} x !llvm.struct<"struct.va_list", (ptr)> {{.*}} : (i32) -> !llvm.ptr %ap = alloca %struct.va_list - ; CHECK: %[[CAST0:.+]] = llvm.bitcast %[[ALLOCA0]] : !llvm.ptr)>> to !llvm.ptr - %ap2 = bitcast %struct.va_list* %ap to i8* - ; CHECK: llvm.intr.vastart %[[CAST0]] - call void @llvm.va_start(i8* %ap2) - - ; CHECK: %[[ALLOCA1:.+]] = llvm.alloca %{{.*}} x !llvm.ptr {{.*}} : (i32) -> !llvm.ptr> - %aq = alloca i8* - ; CHECK: %[[CAST1:.+]] = llvm.bitcast %[[ALLOCA1]] : !llvm.ptr> to !llvm.ptr - %aq2 = bitcast i8** %aq to i8* - ; CHECK: llvm.intr.vacopy %[[CAST0]] to %[[CAST1]] - call void @llvm.va_copy(i8* %aq2, i8* %ap2) - ; CHECK: llvm.intr.vaend %[[CAST1]] - call void @llvm.va_end(i8* %aq2) - - ; CHECK: llvm.intr.vaend %[[CAST0]] - call void @llvm.va_end(i8* %ap2) + ; CHECK: llvm.intr.vastart %[[ALLOCA0]] + call void @llvm.va_start(ptr %ap) + + ; CHECK: %[[ALLOCA1:.+]] = llvm.alloca %{{.*}} x !llvm.ptr {{.*}} : (i32) -> !llvm.ptr + %aq = alloca ptr + ; CHECK: llvm.intr.vacopy %[[ALLOCA0]] to %[[ALLOCA1]] + call void @llvm.va_copy(ptr %aq, ptr %ap) + ; CHECK: llvm.intr.vaend %[[ALLOCA1]] + call void @llvm.va_end(ptr %aq) + + ; CHECK: llvm.intr.vaend %[[ALLOCA0]] + call void @llvm.va_end(ptr %ap) ; CHECK: llvm.return ret void } diff --git a/mlir/test/Target/LLVMIR/Import/constant.ll b/mlir/test/Target/LLVMIR/Import/constant.ll index 8ff836d066c4..85cde7b1bcfe 100644 --- a/mlir/test/Target/LLVMIR/Import/constant.ll +++ b/mlir/test/Target/LLVMIR/Import/constant.ll @@ -1,4 +1,4 @@ -; RUN: mlir-translate -opaque-pointers=0 -import-llvm -split-input-file %s | FileCheck %s +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s ; CHECK-LABEL: @int_constants define void @int_constants(i16 %arg0, i32 %arg1, i1 %arg2) { @@ -48,22 +48,10 @@ define void @undef_constant(i32 %arg0) { ; // ----- ; CHECK-LABEL: @null_constant -define i32* @null_constant() { - ; CHECK: %[[NULL:[0-9]+]] = llvm.mlir.null : !llvm.ptr - ; CHECK: llvm.return %[[NULL]] : !llvm.ptr - ret i32* bitcast (double* null to i32*) -} - -; // ----- - -@global = external global double, align 8 - -; CHECK-LABEL: @bitcast_const_expr -define i32* @bitcast_const_expr() { - ; CHECK: %[[VAL0:.*]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[VAL1:.*]] = llvm.bitcast %[[VAL0]] : !llvm.ptr to !llvm.ptr - ; CHECK: llvm.return %[[VAL1]] : !llvm.ptr - ret i32* bitcast (double* @global to i32*) +define ptr @null_constant() { + ; CHECK: %[[NULL:[0-9]+]] = llvm.mlir.null : !llvm.ptr + ; CHECK: llvm.return %[[NULL]] : !llvm.ptr + ret ptr null } ; // ----- @@ -71,12 +59,12 @@ define i32* @bitcast_const_expr() { @global = external global i32, align 8 ; CHECK-LABEL: @gep_const_expr -define i32* @gep_const_expr() { - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 - ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: llvm.return %[[GEP]] : !llvm.ptr - ret i32* getelementptr (i32, i32* @global, i32 2) +define ptr @gep_const_expr() { + ; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK-DAG: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 + ; CHECK-DAG: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: llvm.return %[[GEP]] : !llvm.ptr + ret ptr getelementptr (i32, ptr @global, i32 2) } ; // ----- @@ -85,16 +73,16 @@ define i32* @gep_const_expr() { ; CHECK-LABEL: @const_expr_with_duplicate define i64 @const_expr_with_duplicate() { - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 - ; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[DUP:[0-9]+]] = llvm.ptrtoint %[[GEP]] : !llvm.ptr to i64 + ; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK-DAG: %[[IDX:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 + ; CHECK-DAG: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: %[[DUP:[0-9]+]] = llvm.ptrtoint %[[GEP]] : !llvm.ptr to i64 ; Verify the duplicate sub expression is converted only once. - ; CHECK: %[[SUM:[0-9]+]] = llvm.add %[[DUP]], %[[DUP]] : i64 - ; CHECK: llvm.return %[[SUM]] : i64 - ret i64 add (i64 ptrtoint (i32* getelementptr (i32, i32* @global, i32 7) to i64), - i64 ptrtoint (i32* getelementptr (i32, i32* @global, i32 7) to i64)) + ; CHECK-DAG: %[[SUM:[0-9]+]] = llvm.add %[[DUP]], %[[DUP]] : i64 + ; CHECK-DAG: llvm.return %[[SUM]] : i64 + ret i64 add (i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64), + i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64)) } ; // ----- @@ -104,30 +92,30 @@ define i64 @const_expr_with_duplicate() { ; CHECK-LABEL: @const_expr_with_aggregate() define i64 @const_expr_with_aggregate() { ; Compute the vector elements. - ; CHECK: %[[VAL1:[0-9]+]] = llvm.mlir.constant(33 : i64) : i64 - ; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: %[[IDX1:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 - ; CHECK: %[[GEP1:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX1]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[VAL2:[0-9]+]] = llvm.ptrtoint %[[GEP1]] : !llvm.ptr to i64 + ; CHECK-DAG: %[[VAL1:[0-9]+]] = llvm.mlir.constant(33 : i64) : i64 + ; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK-DAG: %[[IDX1:[0-9]+]] = llvm.mlir.constant(7 : i32) : i32 + ; CHECK-DAG: %[[GEP1:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX1]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: %[[VAL2:[0-9]+]] = llvm.ptrtoint %[[GEP1]] : !llvm.ptr to i64 ; Fill the vector. - ; CHECK: %[[VEC1:[0-9]+]] = llvm.mlir.undef : vector<2xi64> - ; CHECK: %[[IDX2:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32 - ; CHECK: %[[VEC2:[0-9]+]] = llvm.insertelement %[[VAL1]], %[[VEC1]][%[[IDX2]] : i32] : vector<2xi64> - ; CHECK: %[[IDX3:[0-9]+]] = llvm.mlir.constant(1 : i32) : i32 - ; CHECK: %[[VEC3:[0-9]+]] = llvm.insertelement %[[VAL2]], %[[VEC2]][%[[IDX3]] : i32] : vector<2xi64> - ; CHECK: %[[IDX4:[0-9]+]] = llvm.mlir.constant(42 : i32) : i32 + ; CHECK-DAG: %[[VEC1:[0-9]+]] = llvm.mlir.undef : vector<2xi64> + ; CHECK-DAG: %[[IDX2:[0-9]+]] = llvm.mlir.constant(0 : i32) : i32 + ; CHECK-DAG: %[[VEC2:[0-9]+]] = llvm.insertelement %[[VAL1]], %[[VEC1]][%[[IDX2]] : i32] : vector<2xi64> + ; CHECK-DAG: %[[IDX3:[0-9]+]] = llvm.mlir.constant(1 : i32) : i32 + ; CHECK-DAG: %[[VEC3:[0-9]+]] = llvm.insertelement %[[VAL2]], %[[VEC2]][%[[IDX3]] : i32] : vector<2xi64> + ; CHECK-DAG: %[[IDX4:[0-9]+]] = llvm.mlir.constant(42 : i32) : i32 ; Compute the extract index. - ; CHECK: %[[GEP2:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX4]]] : (!llvm.ptr, i32) -> !llvm.ptr - ; CHECK: %[[IDX5:[0-9]+]] = llvm.ptrtoint %[[GEP2]] : !llvm.ptr to i64 + ; CHECK-DAG: %[[GEP2:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX4]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK-DAG: %[[IDX5:[0-9]+]] = llvm.ptrtoint %[[GEP2]] : !llvm.ptr to i64 ; Extract the vector element. - ; CHECK: %[[ELEM:[0-9]+]] = llvm.extractelement %[[VEC3]][%[[IDX5]] : i64] : vector<2xi64> - ; CHECK: llvm.return %[[ELEM]] : i64 + ; CHECK-DAG: %[[ELEM:[0-9]+]] = llvm.extractelement %[[VEC3]][%[[IDX5]] : i64] : vector<2xi64> + ; CHECK-DAG: llvm.return %[[ELEM]] : i64 ret i64 extractelement ( - <2 x i64> , - i64 ptrtoint (i32* getelementptr (i32, i32* @global, i32 42) to i64)) + <2 x i64> , + i64 ptrtoint (ptr getelementptr (i32, ptr @global, i32 42) to i64)) } ; // ----- @@ -137,13 +125,13 @@ define i64 @const_expr_with_aggregate() { ; Calling a function that has not been defined yet. ; CHECK-LABEL: @function_address_before_def define i32 @function_address_before_def() { - %1 = alloca i32 ()* - ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr> - ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] - store i32 ()* @callee, i32 ()** %1 - ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] - %2 = load i32 ()*, i32 ()** %1 - ; CHECK: llvm.call %[[INDIR]]() + %1 = alloca ptr + ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr + ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] : !llvm.ptr, !llvm.ptr + store ptr @callee, ptr %1 + ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] : !llvm.ptr -> !llvm.ptr + %2 = load ptr, ptr %1 + ; CHECK: llvm.call %[[INDIR]]() : !llvm.ptr, () -> i32 %3 = call i32 %2() ret i32 %3 } @@ -155,13 +143,13 @@ define i32 @callee() { ; Calling a function that has been defined. ; CHECK-LABEL: @function_address_after_def define i32 @function_address_after_def() { - %1 = alloca i32 ()* - ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr> - ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] - store i32 ()* @callee, i32 ()** %1 - ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] - %2 = load i32 ()*, i32 ()** %1 - ; CHECK: llvm.call %[[INDIR]]() + %1 = alloca ptr + ; CHECK: %[[FUN:.*]] = llvm.mlir.addressof @callee : !llvm.ptr + ; CHECK: llvm.store %[[FUN]], %[[PTR:.*]] : !llvm.ptr, !llvm.ptr + store ptr @callee, ptr %1 + ; CHECK: %[[INDIR:.*]] = llvm.load %[[PTR]] : !llvm.ptr -> !llvm.ptr + %2 = load ptr, ptr %1 + ; CHECK: llvm.call %[[INDIR]]() : !llvm.ptr, () -> i32 %3 = call i32 %2() ret i32 %3 } @@ -170,44 +158,44 @@ define i32 @function_address_after_def() { ; Verify the aggregate constant import. -; CHECK: %[[C0:.+]] = llvm.mlir.constant(9 : i32) : i32 -; CHECK: %[[C1:.+]] = llvm.mlir.constant(4 : i8) : i8 -; CHECK: %[[C2:.+]] = llvm.mlir.constant(8 : i16) : i16 -; CHECK: %[[C3:.+]] = llvm.mlir.constant(7 : i32) : i32 -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> -; CHECK: %[[CHAIN0:.+]] = llvm.insertvalue %[[C0]], %[[ROOT]][0] -; CHECK: %[[CHAIN1:.+]] = llvm.insertvalue %[[C1]], %[[CHAIN0]][1] -; CHECK: %[[CHAIN2:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN1]][2] -; CHECK: %[[CHAIN3:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN2]][3] -; CHECK: llvm.return %[[CHAIN3]] +; CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(9 : i32) : i32 +; CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(4 : i8) : i8 +; CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(8 : i16) : i16 +; CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(7 : i32) : i32 +; CHECK-DAG: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> +; CHECK-DAG: %[[CHAIN0:.+]] = llvm.insertvalue %[[C0]], %[[ROOT]][0] +; CHECK-DAG: %[[CHAIN1:.+]] = llvm.insertvalue %[[C1]], %[[CHAIN0]][1] +; CHECK-DAG: %[[CHAIN2:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN1]][2] +; CHECK-DAG: %[[CHAIN3:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN2]][3] +; CHECK-DAG: llvm.return %[[CHAIN3]] %simple_agg_type = type {i32, i8, i16, i32} @simple_agg = global %simple_agg_type {i32 9, i8 4, i16 8, i32 7} -; CHECK: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32 -; CHECK: %[[C2:.+]] = llvm.mlir.constant(2 : i8) : i8 -; CHECK: %[[C3:.+]] = llvm.mlir.constant(3 : i16) : i16 -; CHECK: %[[C4:.+]] = llvm.mlir.constant(4 : i32) : i32 -; CHECK: %[[NESTED:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> -; CHECK: %[[CHAIN0:.+]] = llvm.insertvalue %[[C1]], %[[NESTED]][0] -; CHECK: %[[CHAIN1:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN0]][1] -; CHECK: %[[CHAIN2:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN1]][2] -; CHECK: %[[CHAIN3:.+]] = llvm.insertvalue %[[C4]], %[[CHAIN2]][3] -; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr> -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"nested_agg_type", (struct<"simple_agg_type", (i32, i8, i16, i32)>, ptr>)> -; CHECK: %[[CHAIN4:.+]] = llvm.insertvalue %[[CHAIN3]], %[[ROOT]][0] -; CHECK: %[[CHAIN5:.+]] = llvm.insertvalue %[[NULL]], %[[CHAIN4]][1] -; CHECK: llvm.return %[[CHAIN5]] -%nested_agg_type = type {%simple_agg_type, %simple_agg_type*} -@nested_agg = global %nested_agg_type { %simple_agg_type{i32 1, i8 2, i16 3, i32 4}, %simple_agg_type* null } - -; CHECK: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr> -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.vec<2 x ptr>> -; CHECK: %[[P0:.+]] = llvm.mlir.constant(0 : i32) : i32 -; CHECK: %[[CHAIN0:.+]] = llvm.insertelement %[[NULL]], %[[ROOT]][%[[P0]] : i32] : !llvm.vec<2 x ptr>> -; CHECK: %[[P1:.+]] = llvm.mlir.constant(1 : i32) : i32 -; CHECK: %[[CHAIN1:.+]] = llvm.insertelement %[[NULL]], %[[CHAIN0]][%[[P1]] : i32] : !llvm.vec<2 x ptr>> -; CHECK: llvm.return %[[CHAIN1]] : !llvm.vec<2 x ptr>> -@vector_agg = global <2 x %simple_agg_type*> <%simple_agg_type* null, %simple_agg_type* null> +; CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32 +; CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(2 : i8) : i8 +; CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(3 : i16) : i16 +; CHECK-DAG: %[[C4:.+]] = llvm.mlir.constant(4 : i32) : i32 +; CHECK-DAG: %[[NESTED:.+]] = llvm.mlir.undef : !llvm.struct<"simple_agg_type", (i32, i8, i16, i32)> +; CHECK-DAG: %[[CHAIN0:.+]] = llvm.insertvalue %[[C1]], %[[NESTED]][0] +; CHECK-DAG: %[[CHAIN1:.+]] = llvm.insertvalue %[[C2]], %[[CHAIN0]][1] +; CHECK-DAG: %[[CHAIN2:.+]] = llvm.insertvalue %[[C3]], %[[CHAIN1]][2] +; CHECK-DAG: %[[CHAIN3:.+]] = llvm.insertvalue %[[C4]], %[[CHAIN2]][3] +; CHECK-DAG: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK-DAG: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"nested_agg_type", (struct<"simple_agg_type", (i32, i8, i16, i32)>, ptr)> +; CHECK-DAG: %[[CHAIN4:.+]] = llvm.insertvalue %[[CHAIN3]], %[[ROOT]][0] +; CHECK-DAG: %[[CHAIN5:.+]] = llvm.insertvalue %[[NULL]], %[[CHAIN4]][1] +; CHECK-DAG: llvm.return %[[CHAIN5]] +%nested_agg_type = type {%simple_agg_type, ptr} +@nested_agg = global %nested_agg_type { %simple_agg_type{i32 1, i8 2, i16 3, i32 4}, ptr null } + +; CHECK-DAG: %[[NULL:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK-DAG: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.vec<2 x ptr> +; CHECK-DAG: %[[P0:.+]] = llvm.mlir.constant(0 : i32) : i32 +; CHECK-DAG: %[[CHAIN0:.+]] = llvm.insertelement %[[NULL]], %[[ROOT]][%[[P0]] : i32] : !llvm.vec<2 x ptr> +; CHECK-DAG: %[[P1:.+]] = llvm.mlir.constant(1 : i32) : i32 +; CHECK-DAG: %[[CHAIN1:.+]] = llvm.insertelement %[[NULL]], %[[CHAIN0]][%[[P1]] : i32] : !llvm.vec<2 x ptr> +; CHECK-DAG: llvm.return %[[CHAIN1]] : !llvm.vec<2 x ptr> +@vector_agg = global <2 x ptr> ; // ----- @@ -217,12 +205,24 @@ define i32 @function_address_after_def() { ; CHECK-LABEL: @const_exprs_with_duplicate define i64 @const_exprs_with_duplicate() { - ; CHECK: %[[ADDR:.+]] = llvm.mlir.addressof @global : !llvm.ptr - ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr - %1 = add i64 1, ptrtoint (i32* getelementptr (i32, i32* @global, i32 7) to i64) + ; CHECK: %[[ADDR:.+]] = llvm.mlir.addressof @global : !llvm.ptr + ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr + %1 = add i64 1, ptrtoint (ptr getelementptr (i32, ptr @global, i32 7) to i64) ; Verify the address value is reused. - ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr - %2 = add i64 %1, ptrtoint (i32* getelementptr (i32, i32* @global, i32 42) to i64) + ; CHECK: llvm.getelementptr %[[ADDR]][%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr + %2 = add i64 %1, ptrtoint (ptr getelementptr (i32, ptr @global, i32 42) to i64) ret i64 %2 } + +; // ----- + +; Verify the import of constant expressions with cyclic dependencies. + +@cyclic = internal constant i64 mul (i64 ptrtoint (ptr @cyclic to i64), i64 ptrtoint (ptr @cyclic to i64)) + +; CHECK-LABEL: @cyclic +; CHECK: %[[ADDR:.+]] = llvm.mlir.addressof @cyclic +; CHECK: %[[VAL0:.+]] = llvm.ptrtoint %[[ADDR]] +; CHECK: %[[VAL1:.+]] = llvm.mul %[[VAL0]], %[[VAL0]] +; CHECK: llvm.return %[[VAL1]] diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll index 588eea590de3..a8aae4573e9a 100644 --- a/mlir/test/Target/LLVMIR/Import/debug-info.ll +++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll @@ -112,10 +112,10 @@ define i32 @lexical_block_file(i32 %arg1) { ; // ----- -; CHECK-DAG: #[[VOID:.+]] = #llvm.di_void_result_type +; CHECK-DAG: #[[NULL:.+]] = #llvm.di_null_type ; CHECK-DAG: #[[INT1:.+]] = #llvm.di_basic_type ; CHECK-DAG: #[[INT2:.+]] = #llvm.di_basic_type -; CHECK-DAG: #llvm.di_subroutine_type +; CHECK-DAG: #llvm.di_subroutine_type define void @basic_type() !dbg !3 { ret void @@ -190,7 +190,7 @@ define void @composite_type() !dbg !3 { ; // ----- ; CHECK-DAG: #[[FILE:.+]] = #llvm.di_file<"debug-info.ll" in "/"> -; CHECK-DAG: #[[CU:.+]] = #llvm.di_compile_unit +; CHECK-DAG: #[[CU:.+]] = #llvm.di_compile_unit ; Verify an empty subroutine types list is supported. ; CHECK-DAG: #[[SP_TYPE:.+]] = #llvm.di_subroutine_type ; CHECK-DAG: #[[SP:.+]] = #llvm.di_subprogram @@ -328,3 +328,85 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) !7 = !DILocalVariable(scope: !8, name: "class_field", file: !2, type: !5); !8 = distinct !DISubprogram(name: "class_field", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1) !9 = !DILocation(line: 1, column: 2, scope: !8) + +; // ----- + +; CHECK-DAG: #[[NULL:.+]] = #llvm.di_null_type +; CHECK-DAG: #llvm.di_subroutine_type + +declare !dbg !3 void @variadic_func() + +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2) +!2 = !DIFile(filename: "debug-info.ll", directory: "/") +!3 = !DISubprogram(name: "variadic_func", scope: !2, file: !2, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, type: !4) +!4 = !DISubroutineType(types: !5) +!5 = !{null, null} + +; // ----- + +define void @dbg_use_before_def(ptr %arg) { + call void @llvm.dbg.value(metadata ptr %dbg_arg, metadata !7, metadata !DIExpression()), !dbg !9 + %dbg_arg = getelementptr double, ptr %arg, i64 16 + ret void +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2) +!2 = !DIFile(filename: "debug-info.ll", directory: "/") +!3 = !DICompositeType(tag: DW_TAG_class_type, name: "class_field", file: !2, line: 42, flags: DIFlagTypePassByReference | DIFlagNonTrivial, elements: !4) +!4 = !{!6} +!5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, flags: DIFlagArtificial | DIFlagObjectPointer) +!6 = !DIDerivedType(tag: DW_TAG_member, name: "call_field", file: !2, baseType: !5) +!7 = !DILocalVariable(scope: !8, name: "var", file: !2, type: !5); +!8 = distinct !DISubprogram(name: "dbg_use_before_def", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1) +!9 = !DILocation(line: 1, column: 2, scope: !8) + +; // ----- + +; CHECK-DAG: #[[NAMESPACE:.+]] = #llvm.di_namespace +; CHECK-DAG: #[[SUBPROGRAM:.+]] = #llvm.di_subprogram + +define void @noname_variable(ptr %arg) { + call void @llvm.dbg.value(metadata ptr %arg, metadata !7, metadata !DIExpression()), !dbg !9 + ret void +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!1} +!llvm.module.flags = !{!0} +!0 = !{i32 2, !"Debug Info Version", i32 3} +!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2) +!2 = !DIFile(filename: "debug-info.ll", directory: "/") +!7 = !DILocalVariable(scope: !8) +!8 = distinct !DISubprogram(name: "noname_variable", scope: !2, file: !2, unit: !1); +!9 = !DILocation(line: 1, column: 2, scope: !8) diff --git a/mlir/test/Target/LLVMIR/Import/function-attributes.ll b/mlir/test/Target/LLVMIR/Import/function-attributes.ll index 163020049c79..e70dc7622c76 100644 --- a/mlir/test/Target/LLVMIR/Import/function-attributes.ll +++ b/mlir/test/Target/LLVMIR/Import/function-attributes.ll @@ -29,16 +29,116 @@ attributes #0 = { readnone } ; CHECK-LABEL: @func_arg_attrs ; CHECK-SAME: !llvm.ptr {llvm.byval = i64} ; CHECK-SAME: !llvm.ptr {llvm.byref = i64} -; CHECK-SAME: !llvm.ptr {llvm.sret = i64} -; CHECK-SAME: !llvm.ptr {llvm.inalloca = i64} -define void @func_arg_attrs( +; CHECK-SAME: !llvm.ptr {llvm.noalias} +; CHECK-SAME: !llvm.ptr {llvm.readonly} +; CHECK-SAME: !llvm.ptr {llvm.nest} +; CHECK-SAME: i32 {llvm.signext} +; CHECK-SAME: i64 {llvm.zeroext} +; CHECK-SAME: !llvm.ptr {llvm.align = 64 : i64, llvm.noundef} +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable = 12 : i64} +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable_or_null = 42 : i64} +; CHECK-SAME: f64 {llvm.inreg} +; CHECK-SAME: !llvm.ptr {llvm.nocapture} +; CHECK-SAME: !llvm.ptr {llvm.nofree} +; CHECK-SAME: !llvm.ptr {llvm.nonnull} +; CHECK-SAME: !llvm.ptr {llvm.preallocated = f64} +; CHECK-SAME: !llvm.ptr {llvm.returned} +; CHECK-SAME: !llvm.ptr {llvm.alignstack = 32 : i64} +; CHECK-SAME: !llvm.ptr {llvm.writeonly} +define ptr @func_arg_attrs( ptr byval(i64) %arg0, ptr byref(i64) %arg1, - ptr sret(i64) %arg2, - ptr inalloca(i64) %arg3) { + ptr noalias %arg4, + ptr readonly %arg5, + ptr nest %arg6, + i32 signext %arg7, + i64 zeroext %arg8, + ptr align(64) noundef %arg9, + ptr dereferenceable(12) %arg10, + ptr dereferenceable_or_null(42) %arg11, + double inreg %arg12, + ptr nocapture %arg13, + ptr nofree %arg14, + ptr nonnull %arg15, + ptr preallocated(double) %arg16, + ptr returned %arg17, + ptr alignstack(32) %arg18, + ptr writeonly %arg19) { + ret ptr %arg17 +} + +; CHECK-LABEL: @sret +; CHECK-SAME: !llvm.ptr {llvm.sret = i64} +define void @sret(ptr sret(i64) %arg0) { ret void } +; CHECK-LABEL: @inalloca +; CHECK-SAME: !llvm.ptr {llvm.inalloca = i64} +define void @inalloca(ptr inalloca(i64) %arg0) { + ret void +} + +; CHECK-LABEL: @allocator +; CHECK-SAME: i64 {llvm.allocalign} +; CHECK-SAME: ptr {llvm.allocptr} +declare ptr @allocator(i64 allocalign, ptr allocptr) + +; // ----- + +; CHECK-LABEL: @func_res_attr_noalias +; CHECK-SAME: !llvm.ptr {llvm.noalias} +declare noalias ptr @func_res_attr_noalias() + +; // ----- + +; CHECK-LABEL: @func_res_attr_nonnull +; CHECK-SAME: !llvm.ptr {llvm.nonnull} +declare nonnull ptr @func_res_attr_nonnull() + +; // ----- + +; CHECK-LABEL: @func_res_attr_signext +; CHECK-DAG: llvm.noundef +; CHECK-DAG: llvm.signext +declare noundef signext i32 @func_res_attr_signext() + +; // ----- + +; CHECK-LABEL: @func_res_attr_zeroext +; CHECK-SAME: i32 {llvm.zeroext} +declare zeroext i32 @func_res_attr_zeroext() + +; // ----- + +; CHECK-LABEL: @func_res_attr_align +; CHECK-SAME: !llvm.ptr {llvm.align = 16 : i64} +declare align(16) ptr @func_res_attr_align() + +; // ----- + +; CHECK-LABEL: @func_res_attr_noundef +; CHECK-SAME: !llvm.ptr {llvm.noundef} +declare noundef ptr @func_res_attr_noundef() + +; // ----- + +; CHECK-LABEL: @func_res_attr_dereferenceable +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable = 42 : i64} +declare dereferenceable(42) ptr @func_res_attr_dereferenceable() + +; // ----- + +; CHECK-LABEL: @func_res_attr_dereferenceable_or_null +; CHECK-SAME: !llvm.ptr {llvm.dereferenceable_or_null = 42 : i64} +declare dereferenceable_or_null(42) ptr @func_res_attr_dereferenceable_or_null() + +; // ----- + +; CHECK-LABEL: @func_res_attr_inreg +; CHECK-SAME: !llvm.ptr {llvm.inreg} +declare inreg ptr @func_res_attr_inreg() + ; // ----- ; CHECK-LABEL: @entry_count diff --git a/mlir/test/Target/LLVMIR/Import/global-variables.ll b/mlir/test/Target/LLVMIR/Import/global-variables.ll index 5e68de9b2c01..3b8be65eacd4 100644 --- a/mlir/test/Target/LLVMIR/Import/global-variables.ll +++ b/mlir/test/Target/LLVMIR/Import/global-variables.ll @@ -33,11 +33,10 @@ ; CHECK: llvm.mlir.global internal constant @global_gep_const_expr ; CHECK-SAME: {addr_space = 0 : i32, dso_local} : !llvm.ptr { -; CHECK: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global_int : !llvm.ptr -; CHECK: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 -; CHECK: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr -; CHECK: llvm.return %[[GEP]] : !llvm.ptr -; CHECK: } +; CHECK-DAG: %[[ADDR:[0-9]+]] = llvm.mlir.addressof @global_int : !llvm.ptr +; CHECK-DAG: %[[IDX:[0-9]+]] = llvm.mlir.constant(2 : i32) : i32 +; CHECK-DAG: %[[GEP:[0-9]+]] = llvm.getelementptr %[[ADDR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr +; CHECK-DAG llvm.return %[[GEP]] : !llvm.ptr @global_gep_const_expr = internal constant ptr getelementptr (i32, ptr @global_int, i32 2) ; // ----- diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll index 0570fe4bdeeb..5ef6e991d3c2 100644 --- a/mlir/test/Target/LLVMIR/Import/import-failure.ll +++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll @@ -26,6 +26,7 @@ define i32 @unhandled_value(i32 %arg1) { ; CHECK: import-failure.ll ; CHECK-SAME: error: unhandled instruction: ret ptr blockaddress(@unhandled_constant, %bb1) define ptr @unhandled_constant() { + br label %bb1 bb1: ret ptr blockaddress(@unhandled_constant, %bb1) } @@ -39,6 +40,7 @@ bb1: @private = private global ptr blockaddress(@unhandled_global, %bb1) define void @unhandled_global() { + br label %bb1 bb1: ret void } @@ -48,39 +50,15 @@ bb1: declare void @llvm.gcroot(ptr %arg1, ptr %arg2) ; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled intrinsic: call void @llvm.gcroot(ptr %arg1, ptr %arg2) -define void @unhandled_intrinsic(ptr %arg1, ptr %arg2) { - call void @llvm.gcroot(ptr %arg1, ptr %arg2) +; CHECK-SAME: error: unhandled intrinsic: call void @llvm.gcroot(ptr %arg1, ptr null) +define void @unhandled_intrinsic() gc "example" { + %arg1 = alloca ptr + call void @llvm.gcroot(ptr %arg1, ptr null) ret void } ; // ----- -; CHECK: import-failure.ll -; CHECK-SAME: warning: unhandled metadata: !0 = !{!"unknown metadata"} on br i1 %arg1, label %bb1, label %bb2, !prof !0 -define i64 @unhandled_metadata(i1 %arg1, i64 %arg2) { -entry: - br i1 %arg1, label %bb1, label %bb2, !prof !0 -bb1: - ret i64 %arg2 -bb2: - ret i64 %arg2 -} - -!0 = !{!"unknown metadata"} - -; // ----- - -; CHECK: import-failure.ll -; CHECK-SAME: warning: unhandled function metadata: !0 = !{!"unknown metadata"} on define void @unhandled_func_metadata(i1 %arg1, i64 %arg2) !prof !0 -define void @unhandled_func_metadata(i1 %arg1, i64 %arg2) !prof !0 { - ret void -} - -!0 = !{!"unknown metadata"} - -; // ----- - declare void @llvm.dbg.value(metadata, metadata, metadata) ; CHECK: import-failure.ll @@ -104,10 +82,10 @@ define void @dropped_instruction(i64 %arg1) { ; // ----- -; global_ctors requires the appending linkage type. +; global_dtors with non-null data fields cannot be represented in MLIR. ; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled global variable: @llvm.global_ctors -@llvm.global_ctors = global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr null }] +; CHECK-SAME: error: unhandled global variable: @llvm.global_dtors +@llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr @foo }] define void @foo() { ret void @@ -115,127 +93,232 @@ define void @foo() { ; // ----- -; global_dtors with non-null data fields cannot be represented in MLIR. ; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled global variable: @llvm.global_dtors -@llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @foo, ptr @foo }] +; CHECK-SAME: error: TBAA root node must have non-empty identity: !2 = !{!""} +define dso_local void @tbaa(ptr %0) { + store i8 1, ptr %0, align 4, !tbaa !2 + ret void +} -define void @foo() { +!0 = !{!""} +!1 = !{!"omnipotent char", !0, i64 0} +!2 = !{!1, !1, i64 0} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: error: unsupported TBAA node format: !0 = !{!1, !1, i64 0, i64 4} +define dso_local void @tbaa(ptr %0) { + store i32 1, ptr %0, align 4, !tbaa !2 ret void } +!2 = !{!3, !3, i64 0, i64 4} +!3 = !{!4, i64 4, !"int"} +!4 = !{!5, i64 1, !"omnipotent char"} +!5 = !{!"Simple C++ TBAA"} + ; // ----- -; global_ctors without a data field should not be imported. ; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled global variable: @llvm.global_ctors -@llvm.global_ctors = appending global [1 x { i32, ptr }] [{ i32, ptr } { i32 0, ptr @foo }] +; CHECK-SAME: warning: expected an access group node to be empty and distinct +; CHECK: error: unsupported access group node: !0 = !{} +define void @access_group(ptr %arg1) { + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ret void +} -define void @foo() { +!0 = !{} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected all loop properties to be either debug locations or metadata nodes +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, i32 42} +define void @invalid_loop_node(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } +!0 = distinct !{!0, i32 42} + ; // ----- -; global_dtors with a wrong argument order should not be imported. ; CHECK: import-failure.ll -; CHECK-SAME: error: unhandled global variable: @llvm.global_dtors -@llvm.global_dtors = appending global [1 x { ptr, i32, ptr }] [{ ptr, i32, ptr } { ptr @foo, i32 0, ptr null }] +; CHECK-SAME: warning: cannot import empty loop property +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @invalid_loop_node(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} -define void @foo() { +!0 = distinct !{!0, !1} +!1 = distinct !{} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: cannot import loop property without a name +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @invalid_loop_node(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } +!0 = distinct !{!0, !1} +!1 = distinct !{i1 0} + ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: TBAA root node must have non-empty identity: !2 = !{!""} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !2 +; CHECK-SAME: warning: cannot import loop properties with duplicated names llvm.loop.disable_nonforced +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } -!0 = !{!""} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!1, !1, i64 0} +!0 = distinct !{!0, !1, !1} +!1 = !{!"llvm.loop.disable_nonforced"} ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported TBAA node format: !0 = !{!1, i64 0, i64 0} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !2 +; CHECK-SAME: warning: expected metadata node llvm.loop.disable_nonforced to hold no value +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!1, i64 0, i64 0} +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.disable_nonforced", i1 0} ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: operand '1' must be MDNode: !1 = !{!"omnipotent char", i64 0, i64 0} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !2 +; CHECK-SAME: warning: expected metadata nodes llvm.loop.unroll.enable and llvm.loop.unroll.disable to be mutually exclusive +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !2} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", i64 0, i64 0} -!2 = !{!1, !1, i64 0} +!0 = distinct !{!0, !1, !2} +!1 = !{!"llvm.loop.unroll.enable"} +!2 = !{!"llvm.loop.unroll.disable"} ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: missing member offset: !1 = !{!"agg_t", !2, i64 0, !2} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !3 +; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.enable to hold a boolean value +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!"agg_t", !1, i64 0, !1} -!3 = !{!2, !1, i64 0} +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.vectorize.enable"} ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: operand '4' must be ConstantInt: !1 = !{!"agg_t", !2, i64 0, !2, double 1.000000e+00} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !3 +; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.width to hold an i32 value +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!"agg_t", !1, i64 0, !1, double 1.0} -!3 = !{!2, !1, i64 0} +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.vectorize.width", !0} ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: operand '3' must be ConstantInt: !0 = !{!1, !1, i64 0, double 1.000000e+00} -define dso_local void @tbaa(ptr %0) { - store i8 1, ptr %0, align 4, !tbaa !2 +; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.followup_all to hold an MDNode +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!"omnipotent char", !0, i64 0} -!2 = !{!1, !1, i64 0, double 1.0} +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.vectorize.followup_all", i32 42} ; // ----- ; CHECK: import-failure.ll -; CHECK-SAME: error: unsupported TBAA node format: !0 = !{!1, !1, i64 0, i64 4} -define dso_local void @tbaa(ptr %0) { - store i32 1, ptr %0, align 4, !tbaa !2 +; CHECK-SAME: warning: expected metadata node llvm.loop.parallel_accesses to hold one or multiple MDNodes +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: ret void } -!2 = !{!3, !3, i64 0, i64 4} -!3 = !{!4, i64 4, !"int"} -!4 = !{!5, i64 1, !"omnipotent char"} -!5 = !{!"Simple C++ TBAA"} +!0 = distinct !{!0, !1} +!1 = !{!"llvm.loop.parallel_accesses", i32 42} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: unknown loop annotation llvm.loop.typo +; CHECK: import-failure.ll +; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !2} +define void @unsupported_loop_annotation(i64 %n, ptr %A) { +entry: + br label %end, !llvm.loop !0 +end: + ret void +} + +!0 = distinct !{!0, !1, !2} +!1 = !{!"llvm.loop.disable_nonforced"} +!2 = !{!"llvm.loop.typo"} + +; // ----- + +; CHECK: import-failure.ll +; CHECK-SAME: warning: expected function_entry_count to be attached to a function +; CHECK: warning: unhandled metadata: !0 = !{!"function_entry_count", i64 42} +define void @cond_br(i1 %arg) { +entry: + br i1 %arg, label %bb1, label %bb2, !prof !0 +bb1: + ret void +bb2: + ret void +} + +!0 = !{!"function_entry_count", i64 42} diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll b/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll index e430283ed36b..386709b3b09b 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-constant-caching.ll @@ -1,4 +1,4 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s ; Testing the fix for issue where llvm.getelementptr translated from the second ; ConstantExpr-GEP tried to use llvm.constant(0: i32)-s that were below itself, @@ -7,24 +7,24 @@ ; This test is primarily used to make sure an verification is passed. Thus, we ; only wrote minimum level of checks. -%my_struct = type {i32, i8*} -; CHECK: llvm.mlir.constant(8 : i32) : i32 -; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr> -; CHECK: llvm.mlir.constant(0 : i32) : i32 -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.constant(7 : i32) : i32 -; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr> -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.return +%my_struct = type {i32, ptr} +; CHECK-DAG: llvm.mlir.constant(8 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str0 : !llvm.ptr +; CHECK-DAG: llvm.mlir.constant(0 : i32) : i32 +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.constant(7 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str1 : !llvm.ptr +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.return @str0 = private unnamed_addr constant [5 x i8] c"aaaa\00" @str1 = private unnamed_addr constant [5 x i8] c"bbbb\00" -@g = global [2 x %my_struct] [%my_struct {i32 8, i8* getelementptr ([5 x i8], [5 x i8]* @str0, i32 0, i32 0)}, %my_struct {i32 7, i8* getelementptr ([5 x i8], [5 x i8]* @str1, i32 0, i32 0)}] +@g = global [2 x %my_struct] [%my_struct {i32 8, ptr getelementptr ([5 x i8], ptr @str0, i32 0, i32 1)}, %my_struct {i32 7, ptr getelementptr ([5 x i8], ptr @str1, i32 0, i32 1)}] diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll b/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll index 43f6377c9555..75ba51550b1e 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-constexpr-inst-caching.ll @@ -1,30 +1,30 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s ; REQUIRES: asserts ; This test is primarily used to make sure an assertion is not triggered. ; Thus, we only wrote minimum level of checks. -%my_struct = type {i32, i8*} -; CHECK: llvm.mlir.constant(8 : i32) : i32 -; CHECK: llvm.mlir.addressof @str0 : !llvm.ptr> -; CHECK: llvm.mlir.constant(0 : i32) : i32 -; CHECK: llvm.mlir.constant(1 : i32) : i32 -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.constant(7 : i32) : i32 -; CHECK: llvm.mlir.addressof @str1 : !llvm.ptr> -; CHECK: llvm.mlir.constant(2 : i32) : i32 -; CHECK: llvm.mlir.constant(3 : i32) : i32 -; CHECK: llvm.getelementptr -; CHECK: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> -; CHECK: llvm.insertvalue -; CHECK: llvm.insertvalue -; CHECK: llvm.return +%my_struct = type {i32, ptr} +; CHECK-DAG: llvm.mlir.constant(8 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str0 : !llvm.ptr +; CHECK-DAG: llvm.mlir.constant(0 : i32) : i32 +; CHECK-DAG: llvm.mlir.constant(1 : i32) : i32 +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.constant(7 : i32) : i32 +; CHECK-DAG: llvm.mlir.addressof @str1 : !llvm.ptr +; CHECK-DAG: llvm.mlir.constant(2 : i32) : i32 +; CHECK-DAG: llvm.mlir.constant(3 : i32) : i32 +; CHECK-DAG: llvm.getelementptr +; CHECK-DAG: llvm.mlir.undef : !llvm.struct<"my_struct", (i32, ptr)> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.mlir.undef : !llvm.array<2 x struct<"my_struct", (i32, ptr)>> +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.insertvalue +; CHECK-DAG: llvm.return @str0 = private unnamed_addr constant [5 x i8] c"aaaa\00" @str1 = private unnamed_addr constant [5 x i8] c"bbbb\00" -@g = global [2 x %my_struct] [%my_struct {i32 8, i8* getelementptr ([5 x i8], [5 x i8]* @str0, i32 0, i32 1)}, %my_struct {i32 7, i8* getelementptr ([5 x i8], [5 x i8]* @str1, i32 2, i32 3)}] +@g = global [2 x %my_struct] [%my_struct {i32 8, ptr getelementptr ([5 x i8], ptr @str0, i32 0, i32 1)}, %my_struct {i32 7, ptr getelementptr ([5 x i8], ptr @str1, i32 2, i32 3)}] diff --git a/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll b/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll index 7de99158cc02..6bf7572e7e79 100644 --- a/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll +++ b/mlir/test/Target/LLVMIR/Import/incorrect-scalable-vector-check.ll @@ -1,8 +1,8 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s ; CHECK: llvm.func @shufflevector_crash -define void @shufflevector_crash(<2 x i32*> %arg0) { - ; CHECK: llvm.shufflevector %{{.+}}, %{{.+}} [1, 0] : !llvm.vec<2 x ptr> - %1 = shufflevector <2 x i32*> %arg0, <2 x i32*> undef, <2 x i32> +define void @shufflevector_crash(<2 x ptr> %arg0) { + ; CHECK: llvm.shufflevector %{{.+}}, %{{.+}} [1, 0] : !llvm.vec<2 x ptr> + %1 = shufflevector <2 x ptr> %arg0, <2 x ptr> undef, <2 x i32> ret void } diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index 329605cce790..47076a7abb01 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -251,7 +251,7 @@ define void @integer_arith(i32 %arg1, i32 %arg2, i64 %arg3, i64 %arg4) { ; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[IDX:[a-zA-Z0-9]+]] define half @extract_element(ptr %vec, i32 %idx) { - ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] : !llvm.ptr -> vector<4xf16> + ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] {{.*}} : !llvm.ptr -> vector<4xf16> ; CHECK: %[[V2:.+]] = llvm.extractelement %[[V1]][%[[IDX]] : i32] : vector<4xf16> ; CHECK: llvm.return %[[V2]] %1 = load <4 x half>, ptr %vec @@ -266,7 +266,7 @@ define half @extract_element(ptr %vec, i32 %idx) { ; CHECK-SAME: %[[VAL:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[IDX:[a-zA-Z0-9]+]] define <4 x half> @insert_element(ptr %vec, half %val, i32 %idx) { - ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] : !llvm.ptr -> vector<4xf16> + ; CHECK: %[[V1:.+]] = llvm.load %[[VEC]] {{.*}} : !llvm.ptr -> vector<4xf16> ; CHECK: %[[V2:.+]] = llvm.insertelement %[[VAL]], %[[V1]][%[[IDX]] : i32] : vector<4xf16> ; CHECK: llvm.return %[[V2]] %1 = load <4 x half>, ptr %vec @@ -352,10 +352,34 @@ define ptr @alloca(i64 %size) { ; CHECK-LABEL: @load_store ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] define void @load_store(ptr %ptr) { - ; CHECK: %[[V1:[0-9]+]] = llvm.load %[[PTR]] : !llvm.ptr -> f64 - ; CHECK: llvm.store %[[V1]], %[[PTR]] : f64, !llvm.ptr + ; CHECK: %[[V1:[0-9]+]] = llvm.load %[[PTR]] {alignment = 8 : i64} : !llvm.ptr -> f64 + ; CHECK: %[[V2:[0-9]+]] = llvm.load volatile %[[PTR]] {alignment = 16 : i64, nontemporal} : !llvm.ptr -> f64 %1 = load double, ptr %ptr + %2 = load volatile double, ptr %ptr, align 16, !nontemporal !0 + + ; CHECK: llvm.store %[[V1]], %[[PTR]] {alignment = 8 : i64} : f64, !llvm.ptr + ; CHECK: llvm.store volatile %[[V2]], %[[PTR]] {alignment = 16 : i64, nontemporal} : f64, !llvm.ptr store double %1, ptr %ptr + store volatile double %2, ptr %ptr, align 16, !nontemporal !0 + ret void +} + +!0 = !{i32 1} + +; // ----- + +; CHECK-LABEL: @atomic_load_store +; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] +define void @atomic_load_store(ptr %ptr) { + ; CHECK: %[[V1:[0-9]+]] = llvm.load %[[PTR]] atomic acquire {alignment = 8 : i64} : !llvm.ptr -> f64 + ; CHECK: %[[V2:[0-9]+]] = llvm.load volatile %[[PTR]] atomic syncscope("singlethreaded") acquire {alignment = 16 : i64} : !llvm.ptr -> f64 + %1 = load atomic double, ptr %ptr acquire, align 8 + %2 = load atomic volatile double, ptr %ptr syncscope("singlethreaded") acquire, align 16 + + ; CHECK: llvm.store %[[V1]], %[[PTR]] atomic release {alignment = 8 : i64} : f64, !llvm.ptr + ; CHECK: llvm.store volatile %[[V2]], %[[PTR]] atomic syncscope("singlethreaded") release {alignment = 16 : i64} : f64, !llvm.ptr + store atomic double %1, ptr %ptr release, align 8 + store atomic volatile double %2, ptr %ptr syncscope("singlethreaded") release, align 16 ret void } @@ -367,32 +391,45 @@ define void @load_store(ptr %ptr) { ; CHECK-SAME: %[[PTR2:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[VAL2:[a-zA-Z0-9]+]] define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) { - ; CHECK: llvm.atomicrmw xchg %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw xchg %[[PTR1]], %[[VAL1]] acquire %1 = atomicrmw xchg ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw add %[[PTR1]], %[[VAL1]] release : i32 + ; CHECK: llvm.atomicrmw add %[[PTR1]], %[[VAL1]] release %2 = atomicrmw add ptr %ptr1, i32 %val1 release - ; CHECK: llvm.atomicrmw sub %[[PTR1]], %[[VAL1]] acq_rel : i32 + ; CHECK: llvm.atomicrmw sub %[[PTR1]], %[[VAL1]] acq_rel %3 = atomicrmw sub ptr %ptr1, i32 %val1 acq_rel - ; CHECK: llvm.atomicrmw _and %[[PTR1]], %[[VAL1]] seq_cst : i32 + ; CHECK: llvm.atomicrmw _and %[[PTR1]], %[[VAL1]] seq_cst %4 = atomicrmw and ptr %ptr1, i32 %val1 seq_cst - ; CHECK: llvm.atomicrmw nand %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw nand %[[PTR1]], %[[VAL1]] acquire %5 = atomicrmw nand ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw _or %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw _or %[[PTR1]], %[[VAL1]] acquire %6 = atomicrmw or ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw _xor %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw _xor %[[PTR1]], %[[VAL1]] acquire %7 = atomicrmw xor ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw max %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw max %[[PTR1]], %[[VAL1]] acquire %8 = atomicrmw max ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw min %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw min %[[PTR1]], %[[VAL1]] acquire %9 = atomicrmw min ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw umax %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw umax %[[PTR1]], %[[VAL1]] acquire %10 = atomicrmw umax ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw umin %[[PTR1]], %[[VAL1]] acquire : i32 + ; CHECK: llvm.atomicrmw umin %[[PTR1]], %[[VAL1]] acquire %11 = atomicrmw umin ptr %ptr1, i32 %val1 acquire - ; CHECK: llvm.atomicrmw fadd %[[PTR2]], %[[VAL2]] acquire : f32 + ; CHECK: llvm.atomicrmw fadd %[[PTR2]], %[[VAL2]] acquire %12 = atomicrmw fadd ptr %ptr2, float %val2 acquire - ; CHECK: llvm.atomicrmw fsub %[[PTR2]], %[[VAL2]] acquire : f32 + ; CHECK: llvm.atomicrmw fsub %[[PTR2]], %[[VAL2]] acquire %13 = atomicrmw fsub ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw fmax %[[PTR2]], %[[VAL2]] acquire + %14 = atomicrmw fmax ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw fmin %[[PTR2]], %[[VAL2]] acquire + %15 = atomicrmw fmin ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw uinc_wrap %[[PTR1]], %[[VAL1]] acquire + %16 = atomicrmw uinc_wrap ptr %ptr1, i32 %val1 acquire + ; CHECK: llvm.atomicrmw udec_wrap %[[PTR1]], %[[VAL1]] acquire + %17 = atomicrmw udec_wrap ptr %ptr1, i32 %val1 acquire + + ; CHECK: llvm.atomicrmw volatile + ; CHECK-SAME: syncscope("singlethread") + ; CHECK-SAME: {alignment = 8 : i64} + %18 = atomicrmw volatile udec_wrap ptr %ptr1, i32 %val1 syncscope("singlethread") acquire, align 8 ret void } @@ -403,10 +440,15 @@ define void @atomic_rmw(ptr %ptr1, i32 %val1, ptr %ptr2, float %val2) { ; CHECK-SAME: %[[VAL1:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[VAL2:[a-zA-Z0-9]+]] define void @atomic_cmpxchg(ptr %ptr1, i32 %val1, i32 %val2) { - ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] seq_cst seq_cst : i32 + ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] seq_cst seq_cst %1 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 seq_cst seq_cst - ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] monotonic seq_cst : i32 + ; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] monotonic seq_cst %2 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 monotonic seq_cst + + ; CHECK: llvm.cmpxchg weak volatile + ; CHECK-SAME: syncscope("singlethread") + ; CHECK-SAME: {alignment = 8 : i64} + %3 = cmpxchg weak volatile ptr %ptr1, i32 %val1, i32 %val2 syncscope("singlethread") monotonic seq_cst, align 8 ret void } @@ -415,9 +457,9 @@ define void @atomic_cmpxchg(ptr %ptr1, i32 %val1, i32 %val2) { ; CHECK: llvm.func @fn(i32) -> f32 declare float @fn(i32) -; CHECK-LABEL: @call_fn +; CHECK-LABEL: @direct_call ; CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] -define float @call_fn(i32 %arg1) { +define float @direct_call(i32 %arg1) { ; CHECK: llvm.call @fn(%[[ARG1]]) %1 = call float @fn(i32 %arg1) ret float %1 @@ -425,12 +467,12 @@ define float @call_fn(i32 %arg1) { ; // ----- -; CHECK-LABEL: @call_fn_ptr +; CHECK-LABEL: @indirect_call ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] -define void @call_fn_ptr(ptr %fn) { +define void @indirect_call(ptr addrspace(42) %fn) { ; CHECK: %[[C0:[0-9]+]] = llvm.mlir.constant(0 : i16) : i16 - ; CHECK: llvm.call %[[PTR]](%[[C0]]) - call void %fn(i16 0) + ; CHECK: llvm.call %[[PTR]](%[[C0]]) : !llvm.ptr<42>, (i16) -> () + call addrspace(42) void %fn(i16 0) ret void } @@ -440,13 +482,26 @@ define void @call_fn_ptr(ptr %fn) { ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] define void @gep_static_idx(ptr %ptr) { ; CHECK: %[[IDX:.+]] = llvm.mlir.constant(7 : i32) - ; CHECK: llvm.getelementptr inbounds %[[PTR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr + ; CHECK: llvm.getelementptr inbounds %[[PTR]][%[[IDX]]] : (!llvm.ptr, i32) -> !llvm.ptr, f32 %1 = getelementptr inbounds float, ptr %ptr, i32 7 ret void } ; // ----- +; CHECK: @varargs(...) +declare void @varargs(...) + +; CHECK-LABEL: @varargs_call +; CHECK-SAME: %[[ARG1:[a-zA-Z0-9]+]] +define void @varargs_call(i32 %0) { + ; CHECK: llvm.call @varargs(%[[ARG1]]) : (i32) -> () + call void (...) @varargs(i32 %0) + ret void +} + +; // ----- + %sub_struct = type { i32, i8 } %my_struct = type { %sub_struct, [4 x i32] } @@ -455,7 +510,7 @@ define void @gep_static_idx(ptr %ptr) { ; CHECK-SAME: %[[IDX:[a-zA-Z0-9]+]] define void @gep_dynamic_idx(ptr %ptr, i32 %idx) { ; CHECK: %[[C0:.+]] = llvm.mlir.constant(0 : i32) - ; CHECK: llvm.getelementptr %[[PTR]][%[[C0]], 1, %[[IDX]]] + ; CHECK: llvm.getelementptr %[[PTR]][%[[C0]], 1, %[[IDX]]]{{.*}}"my_struct" %1 = getelementptr %my_struct, ptr %ptr, i32 0, i32 1, i32 %idx ret void } diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll index 7f225a3e2b44..9bceb000b104 100644 --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -1,7 +1,7 @@ -; RUN: mlir-translate -opaque-pointers=0 -import-llvm %s | FileCheck %s +; RUN: mlir-translate -import-llvm %s | FileCheck %s ; CHECK-LABEL: llvm.func @fmuladd_test -define void @fmuladd_test(float %0, float %1, <8 x float> %2, i8* %3) { +define void @fmuladd_test(float %0, float %1, <8 x float> %2, ptr %3) { ; CHECK: llvm.intr.fmuladd(%{{.*}}, %{{.*}}, %{{.*}}) : (f32, f32, f32) -> f32 %5 = call float @llvm.fmuladd.f32(float %0, float %1, float %0) ; CHECK: llvm.intr.fmuladd(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xf32>, vector<8xf32>, vector<8xf32>) -> vector<8xf32> @@ -10,8 +10,8 @@ define void @fmuladd_test(float %0, float %1, <8 x float> %2, i8* %3) { %7 = call float @llvm.fma.f32(float %0, float %1, float %0) ; CHECK: llvm.intr.fma(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xf32>, vector<8xf32>, vector<8xf32>) -> vector<8xf32> %8 = call <8 x float> @llvm.fma.v8f32(<8 x float> %2, <8 x float> %2, <8 x float> %2) - ; CHECK: "llvm.intr.prefetch"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, i32, i32) -> () - call void @llvm.prefetch.p0i8(i8* %3, i32 0, i32 3, i32 1) + ; CHECK: "llvm.intr.prefetch"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, i32, i32) -> () + call void @llvm.prefetch.p0(ptr %3, i32 0, i32 3, i32 1) ret void } @@ -125,6 +125,14 @@ define void @bitreverse_test(i32 %0, <8 x i32> %1) { %4 = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %1) ret void } +; CHECK-LABEL: llvm.func @byteswap_test +define void @byteswap_test(i32 %0, <8 x i32> %1) { + ; CHECK: llvm.intr.bswap(%{{.*}}) : (i32) -> i32 + %3 = call i32 @llvm.bswap.i32(i32 %0) + ; CHECK: llvm.intr.bswap(%{{.*}}) : (vector<8xi32>) -> vector<8xi32> + %4 = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %1) + ret void +} ; CHECK-LABEL: llvm.func @ctlz_test define void @ctlz_test(i32 %0, <8 x i32> %1) { ; CHECK: %[[FALSE:.+]] = llvm.mlir.constant(false) : i1 @@ -153,6 +161,24 @@ define void @ctpop_test(i32 %0, <8 x i32> %1) { ret void } +; CHECK-LABEL: llvm.func @fshl_test +define void @fshl_test(i32 %0, i32 %1, i32 %2, <8 x i32> %3, <8 x i32> %4, <8 x i32> %5) { + ; CHECK: llvm.intr.fshl(%{{.*}}, %{{.*}}, %{{.*}}) : (i32, i32, i32) -> i32 + %7 = call i32 @llvm.fshl.i32(i32 %0, i32 %1, i32 %2) + ; CHECK: llvm.intr.fshl(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %8 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %3, <8 x i32> %4, <8 x i32> %5) + ret void +} + +; CHECK-LABEL: llvm.func @fshr_test +define void @fshr_test(i32 %0, i32 %1, i32 %2, <8 x i32> %3, <8 x i32> %4, <8 x i32> %5) { + ; CHECK: llvm.intr.fshr(%{{.*}}, %{{.*}}, %{{.*}}) : (i32, i32, i32) -> i32 + %7 = call i32 @llvm.fshr.i32(i32 %0, i32 %1, i32 %2) + ; CHECK: llvm.intr.fshr(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %8 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %3, <8 x i32> %4, <8 x i32> %5) + ret void +} + ; CHECK-LABEL: llvm.func @maximum_test define void @maximum_test(float %0, float %1, <8 x float> %2, <8 x float> %3) { ; CHECK: llvm.intr.maximum(%{{.*}}, %{{.*}}) : (f32, f32) -> f32 @@ -265,7 +291,7 @@ define void @vector_reductions(float %0, <8 x float> %1, <8 x i32> %2) { ; CHECK-SAME: %[[VEC2:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[STRIDE:[a-zA-Z0-9]+]] -define void @matrix_intrinsics(<64 x float> %vec1, <48 x float> %vec2, float* %ptr, i64 %stride) { +define void @matrix_intrinsics(<64 x float> %vec1, <48 x float> %vec2, ptr %ptr, i64 %stride) { ; CHECK: llvm.intr.matrix.multiply %[[VEC1]], %[[VEC2]] ; CHECK-SAME: {lhs_columns = 16 : i32, lhs_rows = 4 : i32, rhs_columns = 3 : i32} %1 = call <12 x float> @llvm.matrix.multiply.v12f32.v64f32.v48f32(<64 x float> %vec1, <48 x float> %vec2, i32 4, i32 16, i32 3) @@ -274,10 +300,10 @@ define void @matrix_intrinsics(<64 x float> %vec1, <48 x float> %vec2, float* %p %2 = call <48 x float> @llvm.matrix.transpose.v48f32(<48 x float> %vec2, i32 3, i32 16) ; CHECK: %[[VAL1:.+]] = llvm.intr.matrix.column.major.load %[[PTR]], ; CHECK-SAME: {columns = 16 : i32, isVolatile = false, rows = 3 : i32} - %3 = call <48 x float> @llvm.matrix.column.major.load.v48f32.i64(float* align 4 %ptr, i64 %stride, i1 false, i32 3, i32 16) + %3 = call <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr align 4 %ptr, i64 %stride, i1 false, i32 3, i32 16) ; CHECK: llvm.intr.matrix.column.major.store %[[VAL1]], %[[PTR]], ; CHECK-SAME: {columns = 16 : i32, isVolatile = true, rows = 3 : i32} - call void @llvm.matrix.column.major.store.v48f32.i64(<48 x float> %3, float* align 4 %ptr, i64 %stride, i1 true, i32 3, i32 16) + call void @llvm.matrix.column.major.store.v48f32.i64(<48 x float> %3, ptr align 4 %ptr, i64 %stride, i1 true, i32 3, i32 16) ret void } @@ -291,68 +317,79 @@ define <7 x i1> @get_active_lane_mask(i64 %0, i64 %1) { ; CHECK-LABEL: @masked_load_store_intrinsics ; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]] -define void @masked_load_store_intrinsics(<7 x float>* %vec, <7 x i1> %mask) { +define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) { ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32} - ; CHECK-SAME: (!llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %1 = call <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>* %vec, i32 1, <7 x i1> %mask, <7 x float> undef) + ; CHECK-SAME: (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 1, <7 x i1> %mask, <7 x float> undef) ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32} - %2 = call <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>* %vec, i32 4, <7 x i1> %mask, <7 x float> %1) + %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1) ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32} - ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr> - call void @llvm.masked.store.v7f32.p0v7f32(<7 x float> %2, <7 x float>* %vec, i32 8, <7 x i1> %mask) + ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr + call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask) ret void } ; CHECK-LABEL: @masked_gather_scatter_intrinsics ; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]] ; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]] -define void @masked_gather_scatter_intrinsics(<7 x float*> %vec, <7 x i1> %mask) { +define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) { ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32} - ; CHECK-SAME: (!llvm.vec<7 x ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %vec, i32 1, <7 x i1> %mask, <7 x float> undef) + ; CHECK-SAME: (!llvm.vec<7 x ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 1, <7 x i1> %mask, <7 x float> undef) ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32} - %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %vec, i32 4, <7 x i1> %mask, <7 x float> %1) + %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 4, <7 x i1> %mask, <7 x float> %1) ; CHECK: llvm.intr.masked.scatter %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32} - ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.vec<7 x ptr> - call void @llvm.masked.scatter.v7f32.v7p0f32(<7 x float> %2, <7 x float*> %vec, i32 8, <7 x i1> %mask) + ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.vec<7 x ptr> + call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %2, <7 x ptr> %vec, i32 8, <7 x i1> %mask) ret void } ; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics -define void @masked_expand_compress_intrinsics(float* %0, <7 x i1> %1, <7 x float> %2) { - ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %4 = call <7 x float> @llvm.masked.expandload.v7f32(float* %0, <7 x i1> %1, <7 x float> %2) - ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> () - call void @llvm.masked.compressstore.v7f32(<7 x float> %4, float* %0, <7 x i1> %1) +define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> %2) { + ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2) + ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> () + call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr %0, <7 x i1> %1) + ret void +} + +; CHECK-LABEL: llvm.func @trap_intrinsics +define void @trap_intrinsics() { + ; CHECK: "llvm.intr.trap"() : () -> () + call void @llvm.trap() + ; CHECK: "llvm.intr.debugtrap"() : () -> () + call void @llvm.debugtrap() + ; CHECK: "llvm.intr.ubsantrap"() {failureKind = 1 : i8} : () -> () + call void @llvm.ubsantrap(i8 1) ret void } ; CHECK-LABEL: llvm.func @memcpy_test -define void @memcpy_test(i32 %0, i8* %1, i8* %2) { +define void @memcpy_test(i32 %0, ptr %1, ptr %2) { ; CHECK: %[[FALSE:.+]] = llvm.mlir.constant(false) : i1 ; CHECK: %[[CST:.+]] = llvm.mlir.constant(10 : i64) : i64 - ; CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () - call void @llvm.memcpy.p0i8.p0i8.i32(i8* %1, i8* %2, i32 %0, i1 false) - ; CHECK: "llvm.intr.memcpy.inline"(%{{.*}}, %{{.*}}, %[[CST]], %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () - call void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* %1, i8* %2, i64 10, i1 false) + ; CHECK: "llvm.intr.memcpy"(%{{.*}}, %{{.*}}, %{{.*}}, %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () + call void @llvm.memcpy.p0.p0.i32(ptr %1, ptr %2, i32 %0, i1 false) + ; CHECK: "llvm.intr.memcpy.inline"(%{{.*}}, %{{.*}}, %[[CST]], %[[FALSE]]) : (!llvm.ptr, !llvm.ptr, i64, i1) -> () + call void @llvm.memcpy.inline.p0.p0.i64(ptr %1, ptr %2, i64 10, i1 false) ret void } ; CHECK-LABEL: llvm.func @memmove_test -define void @memmove_test(i32 %0, i8* %1, i8* %2) { +define void @memmove_test(i32 %0, ptr %1, ptr %2) { ; CHECK: %[[falseval:.+]] = llvm.mlir.constant(false) : i1 - ; CHECK: "llvm.intr.memmove"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () - call void @llvm.memmove.p0i8.p0i8.i32(i8* %1, i8* %2, i32 %0, i1 false) + ; CHECK: "llvm.intr.memmove"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, !llvm.ptr, i32, i1) -> () + call void @llvm.memmove.p0.p0.i32(ptr %1, ptr %2, i32 %0, i1 false) ret void } ; CHECK-LABEL: llvm.func @memset_test -define void @memset_test(i32 %0, i8* %1, i8 %2) { +define void @memset_test(i32 %0, ptr %1, i8 %2) { ; CHECK: %[[falseval:.+]] = llvm.mlir.constant(false) : i1 - ; CHECK: "llvm.intr.memset"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, i8, i32, i1) -> () - call void @llvm.memset.p0i8.i32(i8* %1, i8 %2, i32 %0, i1 false) + ; CHECK: "llvm.intr.memset"(%{{.*}}, %{{.*}}, %{{.*}}, %[[falseval]]) : (!llvm.ptr, i8, i32, i1) -> () + call void @llvm.memset.p0.i32(ptr %1, i8 %2, i32 %0, i1 false) ret void } @@ -410,14 +447,68 @@ define void @umul_with_overflow_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) ret void } +; CHECK-LABEL: llvm.func @sadd_sat_test +define void @sadd_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.sadd.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.sadd.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.sadd.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @uadd_sat_test +define void @uadd_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.uadd.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.uadd.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.uadd.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @ssub_sat_test +define void @ssub_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.ssub.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.ssub.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.ssub.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @usub_sat_test +define void @usub_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.usub.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.usub.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.usub.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @sshl_sat_test +define void @sshl_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.sshl.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.sshl.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.sshl.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.sshl.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + +; CHECK-LABEL: llvm.func @ushl_sat_test +define void @ushl_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) { + ; CHECK: llvm.intr.ushl.sat(%{{.*}}, %{{.*}}) : (i32, i32) -> i32 + %5 = call i32 @llvm.ushl.sat.i32(i32 %0, i32 %1) + ; CHECK: llvm.intr.ushl.sat(%{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + %6 = call <8 x i32> @llvm.ushl.sat.v8i32(<8 x i32> %2, <8 x i32> %3) + ret void +} + ; CHECK-LABEL: llvm.func @va_intrinsics_test -define void @va_intrinsics_test(i8* %0, i8* %1) { +define void @va_intrinsics_test(ptr %0, ptr %1) { ; CHECK: llvm.intr.vastart %{{.*}} - call void @llvm.va_start(i8* %0) + call void @llvm.va_start(ptr %0) ; CHECK: llvm.intr.vacopy %{{.*}} to %{{.*}} - call void @llvm.va_copy(i8* %1, i8* %0) + call void @llvm.va_copy(ptr %1, ptr %0) ; CHECK: llvm.intr.vaend %{{.*}} - call void @llvm.va_end(i8* %0) + call void @llvm.va_end(ptr %0) ret void } @@ -430,18 +521,18 @@ define void @assume(i1 %true) { } ; CHECK-LABEL: llvm.func @coro_id -define void @coro_id(i32 %0, i8* %1) { +define void @coro_id(i32 %0, ptr %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %3 = call token @llvm.coro.id(i32 %0, i8* %1, i8* %1, i8* null) + %3 = call token @llvm.coro.id(i32 %0, ptr %1, ptr %1, ptr null) ret void } ; CHECK-LABEL: llvm.func @coro_begin -define void @coro_begin(i32 %0, i8* %1) { +define void @coro_begin(i32 %0, ptr %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %3 = call token @llvm.coro.id(i32 %0, i8* %1, i8* %1, i8* null) - ; CHECK: llvm.intr.coro.begin %{{.*}}, %{{.*}} : !llvm.ptr - %4 = call i8* @llvm.coro.begin(token %3, i8* %1) + %3 = call token @llvm.coro.id(i32 %0, ptr %1, ptr %1, ptr null) + ; CHECK: llvm.intr.coro.begin %{{.*}}, %{{.*}} : !llvm.ptr + %4 = call ptr @llvm.coro.begin(token %3, ptr %1) ret void } @@ -463,76 +554,76 @@ define void @coro_align() { } ; CHECK-LABEL: llvm.func @coro_save -define void @coro_save(i8* %0) { +define void @coro_save(ptr %0) { ; CHECK: llvm.intr.coro.save %{{.*}} : !llvm.token - %2 = call token @llvm.coro.save(i8* %0) + %2 = call token @llvm.coro.save(ptr %0) ret void } ; CHECK-LABEL: llvm.func @coro_suspend -define void @coro_suspend(i32 %0, i1 %1, i8* %2) { +define void @coro_suspend(i32 %0, i1 %1, ptr %2) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %4 = call token @llvm.coro.id(i32 %0, i8* %2, i8* %2, i8* null) + %4 = call token @llvm.coro.id(i32 %0, ptr %2, ptr %2, ptr null) ; CHECK: llvm.intr.coro.suspend %{{.*}}, %{{.*}} : i8 %5 = call i8 @llvm.coro.suspend(token %4, i1 %1) ret void } ; CHECK-LABEL: llvm.func @coro_end -define void @coro_end(i8* %0, i1 %1) { +define void @coro_end(ptr %0, i1 %1) { ; CHECK: llvm.intr.coro.end - call i1 @llvm.coro.end(i8* %0, i1 %1) + call i1 @llvm.coro.end(ptr %0, i1 %1) ret void } ; CHECK-LABEL: llvm.func @coro_free -define void @coro_free(i32 %0, i8* %1) { +define void @coro_free(i32 %0, ptr %1) { ; CHECK: llvm.intr.coro.id %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.token - %3 = call token @llvm.coro.id(i32 %0, i8* %1, i8* %1, i8* null) - ; CHECK: llvm.intr.coro.free %{{.*}}, %{{.*}} : !llvm.ptr - %4 = call i8* @llvm.coro.free(token %3, i8* %1) + %3 = call token @llvm.coro.id(i32 %0, ptr %1, ptr %1, ptr null) + ; CHECK: llvm.intr.coro.free %{{.*}}, %{{.*}} : !llvm.ptr + %4 = call ptr @llvm.coro.free(token %3, ptr %1) ret void } ; CHECK-LABEL: llvm.func @coro_resume -define void @coro_resume(i8* %0) { +define void @coro_resume(ptr %0) { ; CHECK: llvm.intr.coro.resume %{{.*}} - call void @llvm.coro.resume(i8* %0) + call void @llvm.coro.resume(ptr %0) ret void } ; CHECK-LABEL: llvm.func @eh_typeid_for -define void @eh_typeid_for(i8* %0) { +define void @eh_typeid_for(ptr %0) { ; CHECK: llvm.intr.eh.typeid.for %{{.*}} : i32 - %2 = call i32 @llvm.eh.typeid.for(i8* %0) + %2 = call i32 @llvm.eh.typeid.for(ptr %0) ret void } ; CHECK-LABEL: llvm.func @stack_save() { define void @stack_save() { - ; CHECK: llvm.intr.stacksave : !llvm.ptr - %1 = call i8* @llvm.stacksave() + ; CHECK: llvm.intr.stacksave : !llvm.ptr + %1 = call ptr @llvm.stacksave() ret void } ; CHECK-LABEL: llvm.func @stack_restore -define void @stack_restore(i8* %0) { +define void @stack_restore(ptr %0) { ; CHECK: llvm.intr.stackrestore %{{.*}} - call void @llvm.stackrestore(i8* %0) + call void @llvm.stackrestore(ptr %0) ret void } ; CHECK-LABEL: llvm.func @lifetime -define void @lifetime(i8* %0) { - ; CHECK: llvm.intr.lifetime.start 16, %{{.*}} : !llvm.ptr - call void @llvm.lifetime.start.p0i8(i64 16, i8* %0) - ; CHECK: llvm.intr.lifetime.end 32, %{{.*}} : !llvm.ptr - call void @llvm.lifetime.end.p0i8(i64 32, i8* %0) +define void @lifetime(ptr %0) { + ; CHECK: llvm.intr.lifetime.start 16, %{{.*}} : !llvm.ptr + call void @llvm.lifetime.start.p0(i64 16, ptr %0) + ; CHECK: llvm.intr.lifetime.end 32, %{{.*}} : !llvm.ptr + call void @llvm.lifetime.end.p0(i64 32, ptr %0) ret void } ; CHECK-LABEL: llvm.func @vector_predication_intrinsics -define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x float> %2, <8 x float> %3, <8 x i64> %4, <8 x double> %5, <8 x i32*> %6, i32 %7, float %8, i32* %9, float* %10, <8 x i1> %11, i32 %12) { +define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x float> %2, <8 x float> %3, <8 x i64> %4, <8 x double> %5, <8 x ptr> %6, i32 %7, float %8, ptr %9, ptr %10, <8 x i1> %11, i32 %12) { ; CHECK: "llvm.intr.vp.add"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> %14 = call <8 x i32> @llvm.vp.add.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.sub"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32> @@ -603,14 +694,14 @@ define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x floa %47 = call <8 x i32> @llvm.vp.select.v8i32(<8 x i1> %11, <8 x i32> %0, <8 x i32> %1, i32 %12) ; CHECK: "llvm.intr.vp.merge"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi1>, vector<8xi32>, vector<8xi32>, i32) -> vector<8xi32> %48 = call <8 x i32> @llvm.vp.merge.v8i32(<8 x i1> %11, <8 x i32> %0, <8 x i32> %1, i32 %12) - ; CHECK: "llvm.intr.vp.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, vector<8xi1>, i32) -> () - call void @llvm.vp.store.v8i32.p0i32(<8 x i32> %0, i32* %9, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.load"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<8xi1>, i32) -> vector<8xi32> - %49 = call <8 x i32> @llvm.vp.load.v8i32.p0i32(i32* %9, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.experimental.vp.strided.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, i32, vector<8xi1>, i32) -> () - call void @llvm.experimental.vp.strided.store.v8i32.p0i32.i32(<8 x i32> %0, i32* %9, i32 %7, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.experimental.vp.strided.load"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, vector<8xi1>, i32) -> vector<8xi32> - %50 = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0i32.i32(i32* %9, i32 %7, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, vector<8xi1>, i32) -> () + call void @llvm.vp.store.v8i32.p0(<8 x i32> %0, ptr %9, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.load"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<8xi1>, i32) -> vector<8xi32> + %49 = call <8 x i32> @llvm.vp.load.v8i32.p0(ptr %9, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.experimental.vp.strided.store"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, !llvm.ptr, i32, vector<8xi1>, i32) -> () + call void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32> %0, ptr %9, i32 %7, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.experimental.vp.strided.load"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, i32, vector<8xi1>, i32) -> vector<8xi32> + %50 = call <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr %9, i32 %7, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.trunc"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> vector<8xi32> %51 = call <8 x i32> @llvm.vp.trunc.v8i32.v8i64(<8 x i64> %4, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.zext"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi32>, vector<8xi1>, i32) -> vector<8xi64> @@ -625,10 +716,10 @@ define void @vector_predication_intrinsics(<8 x i32> %0, <8 x i32> %1, <8 x floa %56 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double> %5, <8 x i1> %11, i32 %12) ; CHECK: "llvm.intr.vp.fptosi"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xf64>, vector<8xi1>, i32) -> vector<8xi64> %57 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double> %5, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.ptrtoint"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.vec<8 x ptr>, vector<8xi1>, i32) -> vector<8xi64> - %58 = call <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0i32(<8 x i32*> %6, <8 x i1> %11, i32 %12) - ; CHECK: "llvm.intr.vp.inttoptr"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> !llvm.vec<8 x ptr> - %59 = call <8 x i32*> @llvm.vp.inttoptr.v8p0i32.v8i64(<8 x i64> %4, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.ptrtoint"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.vec<8 x ptr>, vector<8xi1>, i32) -> vector<8xi64> + %58 = call <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr> %6, <8 x i1> %11, i32 %12) + ; CHECK: "llvm.intr.vp.inttoptr"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<8xi64>, vector<8xi1>, i32) -> !llvm.vec<8 x ptr> + %59 = call <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64> %4, <8 x i1> %11, i32 %12) ret void } @@ -636,7 +727,7 @@ declare float @llvm.fmuladd.f32(float, float, float) declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>) declare float @llvm.fma.f32(float, float, float) declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>) -declare void @llvm.prefetch.p0i8(i8* nocapture readonly, i32 immarg, i32 immarg, i32) +declare void @llvm.prefetch.p0(ptr nocapture readonly, i32 immarg, i32 immarg, i32) declare float @llvm.exp.f32(float) declare <8 x float> @llvm.exp.v8f32(<8 x float>) declare float @llvm.exp2.f32(float) @@ -663,12 +754,18 @@ declare float @llvm.pow.f32(float, float) declare <8 x float> @llvm.pow.v8f32(<8 x float>, <8 x float>) declare i32 @llvm.bitreverse.i32(i32) declare <8 x i32> @llvm.bitreverse.v8i32(<8 x i32>) +declare i32 @llvm.bswap.i32(i32) +declare <8 x i32> @llvm.bswap.v8i32(<8 x i32>) declare i32 @llvm.ctlz.i32(i32, i1 immarg) declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1 immarg) declare i32 @llvm.cttz.i32(i32, i1 immarg) declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1 immarg) declare i32 @llvm.ctpop.i32(i32) declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>) +declare i32 @llvm.fshl.i32(i32, i32, i32) +declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) +declare i32 @llvm.fshr.i32(i32, i32, i32) +declare <8 x i32> @llvm.fshr.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) declare float @llvm.maximum.f32(float, float) declare <8 x float> @llvm.maximum.v8f32(<8 x float>, <8 x float>) declare float @llvm.minimum.f32(float, float) @@ -700,19 +797,22 @@ declare float @llvm.vector.reduce.fmul.v8f32(float, <8 x float>) declare i32 @llvm.vector.reduce.xor.v8i32(<8 x i32>) declare <12 x float> @llvm.matrix.multiply.v12f32.v64f32.v48f32(<64 x float>, <48 x float>, i32 immarg, i32 immarg, i32 immarg) declare <48 x float> @llvm.matrix.transpose.v48f32(<48 x float>, i32 immarg, i32 immarg) -declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(float* nocapture, i64, i1 immarg, i32 immarg, i32 immarg) -declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, float* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) +declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr nocapture, i64, i1 immarg, i32 immarg, i32 immarg) +declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) declare <7 x i1> @llvm.get.active.lane.mask.v7i1.i64(i64, i64) -declare <7 x float> @llvm.masked.load.v7f32.p0v7f32(<7 x float>*, i32 immarg, <7 x i1>, <7 x float>) -declare void @llvm.masked.store.v7f32.p0v7f32(<7 x float>, <7 x float>*, i32 immarg, <7 x i1>) -declare <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*>, i32 immarg, <7 x i1>, <7 x float>) -declare void @llvm.masked.scatter.v7f32.v7p0f32(<7 x float>, <7 x float*>, i32 immarg, <7 x i1>) -declare <7 x float> @llvm.masked.expandload.v7f32(float*, <7 x i1>, <7 x float>) -declare void @llvm.masked.compressstore.v7f32(<7 x float>, float*, <7 x i1>) -declare void @llvm.memcpy.p0i8.p0i8.i32(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i32, i1 immarg) -declare void @llvm.memcpy.inline.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64 immarg, i1 immarg) -declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1 immarg) -declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1 immarg) +declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, i32 immarg, <7 x i1>, <7 x float>) +declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, i32 immarg, <7 x i1>) +declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, i32 immarg, <7 x i1>, <7 x float>) +declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>) +declare <7 x float> @llvm.masked.expandload.v7f32(ptr, <7 x i1>, <7 x float>) +declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr, <7 x i1>) +declare void @llvm.trap() +declare void @llvm.debugtrap() +declare void @llvm.ubsantrap(i8 immarg) +declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) +declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64 immarg, i1 immarg) +declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) +declare void @llvm.memset.p0.i32(ptr nocapture writeonly, i8, i32, i1 immarg) declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32>, <8 x i32>) declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) @@ -725,23 +825,35 @@ declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>) declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) declare { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) -declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) -declare i8* @llvm.coro.begin(token, i8* writeonly) +declare i32 @llvm.sadd.sat.i32(i32, i32) +declare <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.uadd.sat.i32(i32, i32) +declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.ssub.sat.i32(i32, i32) +declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.usub.sat.i32(i32, i32) +declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.sshl.sat.i32(i32, i32) +declare <8 x i32> @llvm.sshl.sat.v8i32(<8 x i32>, <8 x i32>) +declare i32 @llvm.ushl.sat.i32(i32, i32) +declare <8 x i32> @llvm.ushl.sat.v8i32(<8 x i32>, <8 x i32>) +declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) +declare ptr @llvm.coro.begin(token, ptr writeonly) declare i64 @llvm.coro.size.i64() declare i32 @llvm.coro.size.i32() declare i64 @llvm.coro.align.i64() declare i32 @llvm.coro.align.i32() -declare token @llvm.coro.save(i8*) +declare token @llvm.coro.save(ptr) declare i8 @llvm.coro.suspend(token, i1) -declare i1 @llvm.coro.end(i8*, i1) -declare i8* @llvm.coro.free(token, i8* nocapture readonly) -declare void @llvm.coro.resume(i8*) -declare i32 @llvm.eh.typeid.for(i8*) -declare i8* @llvm.stacksave() -declare void @llvm.stackrestore(i8*) -declare void @llvm.va_start(i8*) -declare void @llvm.va_copy(i8*, i8*) -declare void @llvm.va_end(i8*) +declare i1 @llvm.coro.end(ptr, i1) +declare ptr @llvm.coro.free(token, ptr nocapture readonly) +declare void @llvm.coro.resume(ptr) +declare i32 @llvm.eh.typeid.for(ptr) +declare ptr @llvm.stacksave() +declare void @llvm.stackrestore(ptr) +declare void @llvm.va_start(ptr) +declare void @llvm.va_copy(ptr, ptr) +declare void @llvm.va_end(ptr) declare <8 x i32> @llvm.vp.add.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.sub.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32) @@ -777,10 +889,10 @@ declare float @llvm.vp.reduce.fmax.v8f32(float, <8 x float>, <8 x i1>, i32) declare float @llvm.vp.reduce.fmin.v8f32(float, <8 x float>, <8 x i1>, i32) declare <8 x i32> @llvm.vp.select.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32) declare <8 x i32> @llvm.vp.merge.v8i32(<8 x i1>, <8 x i32>, <8 x i32>, i32) -declare void @llvm.vp.store.v8i32.p0i32(<8 x i32>, i32* nocapture, <8 x i1>, i32) -declare <8 x i32> @llvm.vp.load.v8i32.p0i32(i32* nocapture, <8 x i1>, i32) -declare void @llvm.experimental.vp.strided.store.v8i32.p0i32.i32(<8 x i32>, i32* nocapture, i32, <8 x i1>, i32) -declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0i32.i32(i32* nocapture, i32, <8 x i1>, i32) +declare void @llvm.vp.store.v8i32.p0(<8 x i32>, ptr nocapture, <8 x i1>, i32) +declare <8 x i32> @llvm.vp.load.v8i32.p0(ptr nocapture, <8 x i1>, i32) +declare void @llvm.experimental.vp.strided.store.v8i32.p0.i32(<8 x i32>, ptr nocapture, i32, <8 x i1>, i32) +declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr nocapture, i32, <8 x i1>, i32) declare <8 x i32> @llvm.vp.trunc.v8i32.v8i64(<8 x i64>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.zext.v8i64.v8i32(<8 x i32>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.sext.v8i64.v8i32(<8 x i32>, <8 x i1>, i32) @@ -788,8 +900,8 @@ declare <8 x float> @llvm.vp.fptrunc.v8f32.v8f64(<8 x double>, <8 x i1>, i32) declare <8 x double> @llvm.vp.fpext.v8f64.v8f32(<8 x float>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double>, <8 x i1>, i32) declare <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double>, <8 x i1>, i32) -declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0i32(<8 x i32*>, <8 x i1>, i32) -declare <8 x i32*> @llvm.vp.inttoptr.v8p0i32.v8i64(<8 x i64>, <8 x i1>, i32) -declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) -declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) +declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr>, <8 x i1>, i32) +declare <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64>, <8 x i1>, i32) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) declare void @llvm.assume(i1) diff --git a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll new file mode 100644 index 000000000000..9db045dde27a --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll @@ -0,0 +1,61 @@ +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.alias_scope_domain @[[DOMAIN:.*]] {description = "The domain"} +; CHECK: llvm.alias_scope @[[$SCOPE0:.*]] {description = "The first scope", domain = @[[DOMAIN]]} +; CHECK: llvm.alias_scope @[[$SCOPE1:.*]] {domain = @[[DOMAIN]]} +; CHECK: llvm.alias_scope @[[$SCOPE2:.*]] {domain = @[[DOMAIN]]} +; CHECK: } + +; CHECK-LABEL: llvm.func @alias_scope +define void @alias_scope(ptr %arg1) { + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]], @__llvm_global_metadata::@[[$SCOPE2]]] + %1 = load i32, ptr %arg1, !alias.scope !4, !noalias !7 + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]], @__llvm_global_metadata::@[[$SCOPE2]]] + %2 = load i32, ptr %arg1, !alias.scope !5, !noalias !8 + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE2]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]], @__llvm_global_metadata::@[[$SCOPE1]]] + %3 = load i32, ptr %arg1, !alias.scope !6, !noalias !9 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1, !0, !"The first scope"} +!2 = distinct !{!2, !0} +!3 = distinct !{!3, !0} +!4 = !{!1} +!5 = !{!2} +!6 = !{!3} +!7 = !{!2, !3} +!8 = !{!1, !3} +!9 = !{!1, !2} + +; // ----- + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.alias_scope_domain @[[DOMAIN0:.*]] {description = "The domain"} +; CHECK: llvm.alias_scope @[[$SCOPE0:.*]] {domain = @[[DOMAIN0]]} +; CHECK: llvm.alias_scope_domain @[[DOMAIN1:.*]] +; CHECK: llvm.alias_scope @[[$SCOPE1:.*]] {domain = @[[DOMAIN1]]} +; CHECK: } + +; CHECK-LABEL: llvm.func @two_domains +define void @two_domains(ptr %arg1) { + ; CHECK: llvm.load + ; CHECK-SAME: alias_scopes = [@__llvm_global_metadata::@[[$SCOPE0]]] + ; CHECK-SAME: noalias_scopes = [@__llvm_global_metadata::@[[$SCOPE1]]] + %1 = load i32, ptr %arg1, !alias.scope !4, !noalias !5 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1} +!2 = !{!2, !0} +!3 = !{!3, !1} +!4 = !{!2} +!5 = !{!3} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll new file mode 100644 index 000000000000..ff2d9c84fc13 --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -0,0 +1,299 @@ +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.access_group @[[$GROUP0:.*]] +; CHECK: llvm.access_group @[[$GROUP1:.*]] +; CHECK: llvm.access_group @[[$GROUP2:.*]] +; CHECK: llvm.access_group @[[$GROUP3:.*]] +; CHECK: } + +; CHECK-LABEL: llvm.func @access_group +define void @access_group(ptr %arg1) { + ; CHECK: access_groups = [@__llvm_global_metadata::@[[$GROUP0]], @__llvm_global_metadata::@[[$GROUP1]]] + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ; CHECK: access_groups = [@__llvm_global_metadata::@[[$GROUP2]], @__llvm_global_metadata::@[[$GROUP0]]] + %2 = load i32, ptr %arg1, !llvm.access.group !1 + ; CHECK: access_groups = [@__llvm_global_metadata::@[[$GROUP3]]] + %3 = load i32, ptr %arg1, !llvm.access.group !2 + ret void +} + +!0 = !{!3, !4} +!1 = !{!5, !3} +!2 = distinct !{} +!3 = distinct !{} +!4 = distinct !{} +!5 = distinct !{} + +; // ----- + +; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @simple +define void @simple(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4} +!2 = !{!"llvm.loop.disable_nonforced"} +!3 = !{!"llvm.loop.mustprogress"} +!4 = !{!"llvm.loop.isvectorized", i32 1} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[VECTORIZE_ATTR:.*]] = #llvm.loop_vectorize +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @vectorize +define void @vectorize(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8} +!2 = !{!"llvm.loop.vectorize.enable", i1 1} +!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 1} +!4 = !{!"llvm.loop.vectorize.scalable.enable", i1 0} +!5 = !{!"llvm.loop.vectorize.width", i32 16} +!6 = !{!"llvm.loop.vectorize.followup_vectorized", !9} +!7 = !{!"llvm.loop.vectorize.followup_epilogue", !9} +!8 = !{!"llvm.loop.vectorize.followup_all", !9} + +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[INTERLEAVE_ATTR:.*]] = #llvm.loop_interleave +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @interleave +define void @interleave(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.interleave.count", i32 8} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unroll +define void @unroll(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8} +!2 = !{!"llvm.loop.unroll.enable"} +!3 = !{!"llvm.loop.unroll.count", i32 16} +!4 = !{!"llvm.loop.unroll.runtime.disable"} +!5 = !{!"llvm.loop.unroll.full"} +!6 = !{!"llvm.loop.unroll.followup_unrolled", !9} +!7 = !{!"llvm.loop.unroll.followup_remainder", !9} +!8 = !{!"llvm.loop.unroll.followup_all", !9} + +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unroll_disable +define void @unroll_disable(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.unroll.disable"} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[UNROLL_AND_JAM_ATTR:.*]] = #llvm.loop_unroll_and_jam +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unroll_and_jam +define void @unroll_and_jam(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8} +!2 = !{!"llvm.loop.unroll_and_jam.enable"} +!3 = !{!"llvm.loop.unroll_and_jam.count", i32 32} +!4 = !{!"llvm.loop.unroll_and_jam.followup_outer", !9} +!5 = !{!"llvm.loop.unroll_and_jam.followup_inner", !9} +!6 = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", !9} +!7 = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", !9} +!8 = !{!"llvm.loop.unroll_and_jam.followup_all", !9} + +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[LICM_ATTR:.*]] = #llvm.loop_licm +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @licm +define void @licm(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3} +!2 = !{!"llvm.licm.disable"} +!3 = !{!"llvm.loop.licm_versioning.disable"} + +; // ----- + +; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation +; CHECK-DAG: #[[DISTRIBUTE_ATTR:.*]] = #llvm.loop_distribute +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @distribute +define void @distribute(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3, !4, !5, !6} +!2 = !{!"llvm.loop.distribute.enable", i1 0} +!3 = !{!"llvm.loop.distribute.followup_coincident", !9} +!4 = !{!"llvm.loop.distribute.followup_sequential", !9} +!5 = !{!"llvm.loop.distribute.followup_fallback", !9} +!6 = !{!"llvm.loop.distribute.followup_all", !9} + +!9 = distinct !{!9, !10} +!10 = !{!"llvm.loop.disable_nonforced"} + +; // ----- + +; CHECK-DAG: #[[PIPELINE_ATTR:.*]] = #llvm.loop_pipeline +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @pipeline +define void @pipeline(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2, !3} +!2 = !{!"llvm.loop.pipeline.disable", i1 0} +!3 = !{!"llvm.loop.pipeline.initiationinterval", i32 2} + +; // ----- + +; CHECK-DAG: #[[PEELED_ATTR:.*]] = #llvm.loop_peeled +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @peeled +define void @peeled(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.peeled.count", i32 5} + +; // ----- + +; CHECK-DAG: #[[UNSWITCH_ATTR:.*]] = #llvm.loop_unswitch +; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK-LABEL: @unswitched +define void @unswitched(i64 %n, ptr %A) { +entry: +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.unswitch.partial.disable"} + +; // ----- + +; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.access_group @[[GROUP0]] + +; CHECK-LABEL: @parallel_accesses +define void @parallel_accesses(ptr %arg) { +entry: + %0 = load i32, ptr %arg, !llvm.access.group !0 +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!0 = distinct !{} +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.parallel_accesses", !0} + +; // ----- + +; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation + +; CHECK: llvm.metadata @__llvm_global_metadata { +; CHECK: llvm.access_group @[[GROUP0]] +; CHECK: llvm.access_group @[[GROUP1]] + +; CHECK-LABEL: @multiple_parallel_accesses +define void @multiple_parallel_accesses(ptr %arg) { +entry: + %0 = load i32, ptr %arg, !llvm.access.group !0 + %1 = load i32, ptr %arg, !llvm.access.group !3 +; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]} + br label %end, !llvm.loop !1 +end: + ret void +} + +!0 = distinct !{} +!1 = distinct !{!1, !2} +!2 = !{!"llvm.loop.parallel_accesses", !0, !3} +!3 = distinct !{} diff --git a/mlir/test/Target/LLVMIR/Import/profiling-metadata.ll b/mlir/test/Target/LLVMIR/Import/metadata-profiling.ll similarity index 100% rename from mlir/test/Target/LLVMIR/Import/profiling-metadata.ll rename to mlir/test/Target/LLVMIR/Import/metadata-profiling.ll diff --git a/mlir/test/Target/LLVMIR/Import/tbaa.ll b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll similarity index 63% rename from mlir/test/Target/LLVMIR/Import/tbaa.ll rename to mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll index 7daf7d2112be..9e3b7c2ca826 100644 --- a/mlir/test/Target/LLVMIR/Import/tbaa.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll @@ -1,34 +1,37 @@ ; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s -// ----- - -; CHECK-LABEL: llvm.metadata @__tbaa { -; CHECK-NEXT: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} -; CHECK-NEXT: llvm.tbaa_tag @[[T0:tbaa_tag_[0-9]+]] {access_type = @[[R0]], base_type = @[[R0]], offset = 0 : i64} -; CHECK-NEXT: llvm.tbaa_root @[[R1:tbaa_root_[0-9]+]] {id = "Other language TBAA"} -; CHECK-NEXT: llvm.tbaa_tag @[[T1:tbaa_tag_[0-9]+]] {access_type = @[[R1]], base_type = @[[R1]], offset = 0 : i64} +; CHECK-LABEL: llvm.metadata @__llvm_global_metadata { +; CHECK-DAG: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} +; CHECK-DAG: llvm.tbaa_type_desc @[[D0:tbaa_type_desc_[0-9]+]] {id = "scalar type", members = {<@[[R0]], 0>}} +; CHECK-DAG: llvm.tbaa_tag @[[T0:tbaa_tag_[0-9]+]] {access_type = @[[D0]], base_type = @[[D0]], offset = 0 : i64} +; CHECK-DAG: llvm.tbaa_root @[[R1:tbaa_root_[0-9]+]] {id = "Other language TBAA"} +; CHECK-DAG: llvm.tbaa_type_desc @[[D1:tbaa_type_desc_[0-9]+]] {id = "other scalar type", members = {<@[[R1]], 0>}} +; CHECK-DAG: llvm.tbaa_tag @[[T1:tbaa_tag_[0-9]+]] {access_type = @[[D1]], base_type = @[[D1]], offset = 0 : i64} ; CHECK-NEXT: } ; CHECK: llvm.func @tbaa1 ; CHECK: llvm.store %{{.*}}, %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T0]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T0]]] ; CHECK-SAME: } : i8, !llvm.ptr ; CHECK: llvm.store %{{.*}}, %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T1]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T1]]] ; CHECK-SAME: } : i8, !llvm.ptr define dso_local void @tbaa1(ptr %0, ptr %1) { - store i8 1, ptr %0, align 4, !tbaa !1 + store i8 1, ptr %0, align 4, !tbaa !0 store i8 1, ptr %1, align 4, !tbaa !3 ret void } -!0 = !{!"Simple C/C++ TBAA"} -!1 = !{!0, !0, i64 0} -!2 = !{!"Other language TBAA"} -!3 = !{!2, !2, i64 0} +!0 = !{!1, !1, i64 0} +!1 = !{!"scalar type", !2, i64 0} +!2 = !{!"Simple C/C++ TBAA"} + +!3 = !{!4, !4, i64 0} +!4 = !{!"other scalar type", !5, i64 0} +!5 = !{!"Other language TBAA"} // ----- -; CHECK-LABEL: llvm.metadata @__tbaa { +; CHECK-LABEL: llvm.metadata @__llvm_global_metadata { ; CHECK-NEXT: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} ; CHECK-NEXT: llvm.tbaa_tag @[[T0:tbaa_tag_[0-9]+]] {access_type = @[[D1:tbaa_type_desc_[0-9]+]], base_type = @[[D2:tbaa_type_desc_[0-9]+]], offset = 8 : i64} ; CHECK-NEXT: llvm.tbaa_type_desc @[[D1]] {id = "long long", members = {<@[[D0:tbaa_type_desc_[0-9]+]], 0>}} @@ -40,10 +43,10 @@ define dso_local void @tbaa1(ptr %0, ptr %1) { ; CHECK-NEXT: } ; CHECK: llvm.func @tbaa2 ; CHECK: llvm.load %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T0]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T0]]] ; CHECK-SAME: } : !llvm.ptr -> i64 ; CHECK: llvm.store %{{.*}}, %{{.*}} { -; CHECK-SAME: tbaa = [@__tbaa::@[[T1]]] +; CHECK-SAME: tbaa = [@__llvm_global_metadata::@[[T1]]] ; CHECK-SAME: } : i32, !llvm.ptr %struct.agg2_t = type { i64, i64 } %struct.agg1_t = type { i32, i32 } diff --git a/mlir/test/Target/LLVMIR/Import/opaque.ll b/mlir/test/Target/LLVMIR/Import/opaque.ll deleted file mode 100644 index 2d20b2c0b771..000000000000 --- a/mlir/test/Target/LLVMIR/Import/opaque.ll +++ /dev/null @@ -1,53 +0,0 @@ -; RUN: mlir-translate -import-llvm -split-input-file -opaque-pointers %s | FileCheck %s - -; CHECK-LABEL: @opaque_ptr_load -define i32 @opaque_ptr_load(ptr %0) { - ; CHECK: = llvm.load %{{.*}} : !llvm.ptr -> i32 - %2 = load i32, ptr %0, align 4 - ret i32 %2 -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_store -define void @opaque_ptr_store(i32 %0, ptr %1) { - ; CHECK: llvm.store %{{.*}}, %{{.*}} : i32, !llvm.ptr - store i32 %0, ptr %1, align 4 - ret void -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_ptr_store -define void @opaque_ptr_ptr_store(ptr %0, ptr %1) { - ; CHECK: llvm.store %{{.*}}, %{{.*}} : !llvm.ptr, !llvm.ptr - store ptr %0, ptr %1, align 8 - ret void -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_alloca -define ptr @opaque_ptr_alloca(i32 %0) { - ; CHECK: = llvm.alloca %{{.*}} x f32 {alignment = 4 : i64} : (i32) -> !llvm.ptr - %2 = alloca float, i32 %0, align 4 - ret ptr %2 -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_gep -define ptr @opaque_ptr_gep(ptr %0, i32 %1) { - ; CHECK: = llvm.getelementptr inbounds %{{.*}}[%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr, f32 - %3 = getelementptr inbounds float, ptr %0, i32 %1 - ret ptr %3 -} - -; // ----- - -; CHECK-LABEL: @opaque_ptr_gep -define ptr @opaque_ptr_gep_struct(ptr %0, i32 %1){ - ; CHECK: = llvm.getelementptr %{{.*}}[%{{.*}}, 0, 1] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<(struct<(f32, f64)>, struct<(i32, i64)>)> - %3 = getelementptr { { float, double }, { i32, i64 } }, ptr %0, i32 %1, i32 0, i32 1 - ret ptr %3 -} diff --git a/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll b/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll index 27a940cd4e85..41933dcbe7cb 100644 --- a/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll +++ b/mlir/test/Target/LLVMIR/Import/zeroinitializer.ll @@ -1,13 +1,12 @@ -; RUN: mlir-translate -opaque-pointers=0 --import-llvm %s | FileCheck %s +; RUN: mlir-translate --import-llvm %s | FileCheck %s -%Domain = type { %Domain**, %Domain* } +%Domain = type { ptr, ptr } ; CHECK: llvm.mlir.global external @D() -; CHECK-SAME: !llvm.struct<"Domain", (ptr>>, ptr>)> -; CHECK: %[[E0:.+]] = llvm.mlir.null : !llvm.ptr>>, ptr>)>>> -; CHECK: %[[E1:.+]] = llvm.mlir.null : !llvm.ptr>>, ptr>)>> -; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"Domain", (ptr>>, ptr>)> +; CHECK-SAME: !llvm.struct<"Domain", (ptr, ptr)> +; CHECK: %[[E0:.+]] = llvm.mlir.null : !llvm.ptr +; CHECK: %[[ROOT:.+]] = llvm.mlir.undef : !llvm.struct<"Domain", (ptr, ptr)> ; CHECK: %[[CHAIN:.+]] = llvm.insertvalue %[[E0]], %[[ROOT]][0] -; CHECK: %[[RES:.+]] = llvm.insertvalue %[[E1]], %[[CHAIN]][1] +; CHECK: %[[RES:.+]] = llvm.insertvalue %[[E0]], %[[CHAIN]][1] ; CHECK: llvm.return %[[RES]] @D = global %Domain zeroinitializer diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir index 50d18eb2dd5f..a628349951e6 100644 --- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir @@ -48,10 +48,16 @@ llvm.func @func_no_debug() { baseType = #si64, flags = Vector, elements = #llvm.di_subrange > -#void = #llvm.di_void_result_type -#spType0 = #llvm.di_subroutine_type +#null = #llvm.di_null_type +#spType0 = #llvm.di_subroutine_type +#toplevel_namespace = #llvm.di_namespace< + name = "toplevel", exportSymbols = true +> +#nested_namespace = #llvm.di_namespace< + name = "nested", scope = #toplevel_namespace, exportSymbols = false +> #sp0 = #llvm.di_subprogram< - compileUnit = #cu, scope = #file, name = "func_with_debug", linkageName = "func_with_debug", + compileUnit = #cu, scope = #nested_namespace, name = "func_with_debug", linkageName = "func_with_debug", file = #file, line = 3, scopeLine = 3, subprogramFlags = "Definition|Optimized", type = #spType0 > #calleeType = #llvm.di_subroutine_type< @@ -66,6 +72,7 @@ llvm.func @func_no_debug() { #blockScope = #llvm.di_lexical_block #variable = #llvm.di_local_variable #variableAddr = #llvm.di_local_variable +#noNameVariable = #llvm.di_local_variable #spType1 = #llvm.di_subroutine_type #sp1 = #llvm.di_subprogram< @@ -83,9 +90,11 @@ llvm.func @func_with_debug(%arg: i64) { // CHECK: call void @llvm.dbg.value(metadata i64 %[[ARG]], metadata ![[VAR_LOC:[0-9]+]], metadata !DIExpression()) // CHECK: call void @llvm.dbg.addr(metadata ptr %[[ALLOC]], metadata ![[ADDR_LOC:[0-9]+]], metadata !DIExpression()) // CHECK: call void @llvm.dbg.declare(metadata ptr %[[ALLOC]], metadata ![[ADDR_LOC]], metadata !DIExpression()) + // CHECK: call void @llvm.dbg.value(metadata i64 %[[ARG]], metadata ![[NO_NAME_VAR:[0-9]+]], metadata !DIExpression()) llvm.intr.dbg.value #variable = %arg : i64 llvm.intr.dbg.addr #variableAddr = %alloc : !llvm.ptr llvm.intr.dbg.declare #variableAddr = %alloc : !llvm.ptr + llvm.intr.dbg.value #noNameVariable= %arg : i64 // CHECK: call void @func_no_debug(), !dbg ![[CALLSITE_LOC:[0-9]+]] llvm.call @func_no_debug() : () -> () loc(callsite("mysource.cc":3:4 at "mysource.cc":5:6)) @@ -113,7 +122,9 @@ llvm.func @empty_types() { // CHECK: ![[CU_LOC:.*]] = distinct !DICompileUnit(language: DW_LANG_C, file: ![[CU_FILE_LOC:.*]], producer: "MLIR", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) // CHECK: ![[CU_FILE_LOC]] = !DIFile(filename: "foo.mlir", directory: "/test/") -// CHECK: ![[FUNC_LOC]] = distinct !DISubprogram(name: "func_with_debug", linkageName: "func_with_debug", scope: ![[CU_FILE_LOC]], file: ![[CU_FILE_LOC]], line: 3, type: ![[FUNC_TYPE:.*]], scopeLine: 3, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: ![[CU_LOC]]) +// CHECK: ![[FUNC_LOC]] = distinct !DISubprogram(name: "func_with_debug", linkageName: "func_with_debug", scope: ![[NESTED_NAMESPACE:.*]], file: ![[CU_FILE_LOC]], line: 3, type: ![[FUNC_TYPE:.*]], scopeLine: 3, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: ![[CU_LOC]]) +// CHECK: ![[NESTED_NAMESPACE]] = !DINamespace(name: "nested", scope: ![[TOPLEVEL_NAMESPACE:.*]]) +// CHECK: ![[TOPLEVEL_NAMESPACE]] = !DINamespace(name: "toplevel", scope: null, exportSymbols: true) // CHECK: ![[FUNC_TYPE]] = !DISubroutineType(cc: DW_CC_normal, types: ![[FUNC_ARGS:.*]]) // CHECK: ![[FUNC_ARGS]] = !{null, ![[ARG_TYPE:.*]], ![[PTR_TYPE:.*]], ![[NAMED_TYPE:.*]], ![[COMPOSITE_TYPE:.*]], ![[VECTOR_TYPE:.*]]} // CHECK: ![[ARG_TYPE]] = !DIBasicType(name: "si64") @@ -131,6 +142,7 @@ llvm.func @empty_types() { // CHECK: ![[VAR_SCOPE]] = distinct !DILexicalBlockFile(scope: ![[FUNC_LOC]], file: ![[CU_FILE_LOC]], discriminator: 0) // CHECK: ![[ADDR_LOC]] = !DILocalVariable(name: "alloc", scope: ![[BLOCK_LOC:.*]]) // CHECK: ![[BLOCK_LOC]] = distinct !DILexicalBlock(scope: ![[FUNC_LOC]]) +// CHECK: ![[NO_NAME_VAR]] = !DILocalVariable(scope: ![[BLOCK_LOC]]) // CHECK-DAG: ![[CALLSITE_LOC]] = !DILocation(line: 3, column: 4, // CHECK-DAG: ![[FILE_LOC]] = !DILocation(line: 1, column: 2, diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir index e6ff83f15ba7..1dc46159dcce 100644 --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -143,6 +143,15 @@ llvm.func @bitreverse_test(%arg0: i32, %arg1: vector<8xi32>) { llvm.return } +// CHECK-LABEL: @byteswap_test +llvm.func @byteswap_test(%arg0: i32, %arg1: vector<8xi32>) { + // CHECK: call i32 @llvm.bswap.i32 + "llvm.intr.bswap"(%arg0) : (i32) -> i32 + // CHECK: call <8 x i32> @llvm.bswap.v8i32 + "llvm.intr.bswap"(%arg1) : (vector<8xi32>) -> vector<8xi32> + llvm.return +} + // CHECK-LABEL: @ctlz_test llvm.func @ctlz_test(%arg0: i32, %arg1: vector<8xi32>) { %i1 = llvm.mlir.constant(false) : i1 @@ -172,6 +181,24 @@ llvm.func @ctpop_test(%arg0: i32, %arg1: vector<8xi32>) { llvm.return } +// CHECK-LABEL: @fshl_test +llvm.func @fshl_test(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: vector<8xi32>, %arg4: vector<8xi32>, %arg5: vector<8xi32>) { + // CHECK: call i32 @llvm.fshl.i32 + "llvm.intr.fshl"(%arg0, %arg1, %arg2) : (i32, i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.fshl.v8i32 + "llvm.intr.fshl"(%arg3, %arg4, %arg5) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @fshr_test +llvm.func @fshr_test(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: vector<8xi32>, %arg4: vector<8xi32>, %arg5: vector<8xi32>) { + // CHECK: call i32 @llvm.fshr.i32 + "llvm.intr.fshr"(%arg0, %arg1, %arg2) : (i32, i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.fshr.v8i32 + "llvm.intr.fshr"(%arg3, %arg4, %arg5) : (vector<8xi32>, vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + // CHECK-LABEL: @maximum_test llvm.func @maximum_test(%arg0: f32, %arg1: f32, %arg2: vector<8xf32>, %arg3: vector<8xf32>) { // CHECK: call float @llvm.maximum.f32 @@ -347,6 +374,17 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector llvm.return } +// CHECK-LABEL: @trap_intrinsics +llvm.func @trap_intrinsics() { + // CHECK: call void @llvm.trap() + "llvm.intr.trap"() : () -> () + // CHECK: call void @llvm.debugtrap() + "llvm.intr.debugtrap"() : () -> () + // CHECK: call void @llvm.ubsantrap(i8 1) + "llvm.intr.ubsantrap"() {failureKind = 1 : i8} : () -> () + llvm.return +} + // CHECK-LABEL: @memcpy_test llvm.func @memcpy_test(%arg0: i32, %arg2: !llvm.ptr, %arg3: !llvm.ptr) { %i1 = llvm.mlir.constant(false) : i1 @@ -428,6 +466,60 @@ llvm.func @umul_with_overflow_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, llvm.return } +// CHECK-LABEL: @sadd_sat_test +llvm.func @sadd_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.sadd.sat.i32 + "llvm.intr.sadd.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.sadd.sat.v8i32 + "llvm.intr.sadd.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @uadd_sat_test +llvm.func @uadd_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.uadd.sat.i32 + "llvm.intr.uadd.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.uadd.sat.v8i32 + "llvm.intr.uadd.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @ssub_sat_test +llvm.func @ssub_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.ssub.sat.i32 + "llvm.intr.ssub.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.ssub.sat.v8i32 + "llvm.intr.ssub.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @usub_sat_test +llvm.func @usub_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.usub.sat.i32 + "llvm.intr.usub.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.usub.sat.v8i32 + "llvm.intr.usub.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @sshl_sat_test +llvm.func @sshl_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.sshl.sat.i32 + "llvm.intr.sshl.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.sshl.sat.v8i32 + "llvm.intr.sshl.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + +// CHECK-LABEL: @ushl_sat_test +llvm.func @ushl_sat_test(%arg0: i32, %arg1: i32, %arg2: vector<8xi32>, %arg3: vector<8xi32>) { + // CHECK: call i32 @llvm.ushl.sat.i32 + "llvm.intr.ushl.sat"(%arg0, %arg1) : (i32, i32) -> i32 + // CHECK: call <8 x i32> @llvm.ushl.sat.v8i32 + "llvm.intr.ushl.sat"(%arg2, %arg3) : (vector<8xi32>, vector<8xi32>) -> vector<8xi32> + llvm.return +} + // CHECK-LABEL: @coro_id llvm.func @coro_id(%arg0: i32, %arg1: !llvm.ptr) { // CHECK: call token @llvm.coro.id @@ -758,6 +850,9 @@ llvm.func @lifetime(%p: !llvm.ptr) { // CHECK-DAG: declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>) // CHECK-DAG: declare <7 x float> @llvm.masked.expandload.v7f32(ptr nocapture, <7 x i1>, <7 x float>) // CHECK-DAG: declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr nocapture, <7 x i1>) +// CHECK-DAG: declare void @llvm.trap() +// CHECK-DAG: declare void @llvm.debugtrap() +// CHECK-DAG: declare void @llvm.ubsantrap(i8 immarg) // CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) // CHECK-DAG: declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64 immarg, i1 immarg) // CHECK-DAG: declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) @@ -770,6 +865,18 @@ llvm.func @lifetime(%p: !llvm.ptr) { // CHECK-DAG: declare { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32>, <8 x i32>) #0 // CHECK-DAG: declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) // CHECK-DAG: declare { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>) #0 +// CHECK-DAG: declare i32 @llvm.sadd.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.uadd.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.ssub.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.usub.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.sshl.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.sshl.sat.v8i32(<8 x i32>, <8 x i32>) +// CHECK-DAG: declare i32 @llvm.ushl.sat.i32(i32, i32) +// CHECK-DAG: declare <8 x i32> @llvm.ushl.sat.v8i32(<8 x i32>, <8 x i32>) // CHECK-DAG: declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) // CHECK-DAG: declare ptr @llvm.coro.begin(token, ptr writeonly) // CHECK-DAG: declare i64 @llvm.coro.size.i64() diff --git a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir index e8571a992e5c..bc06841040fc 100644 --- a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir @@ -7,96 +7,6 @@ func.func @foo() { // ----- -// expected-error @below{{llvm.noalias attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_noalias(%arg0 : f32 {llvm.noalias}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.sret attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_sret(%arg0 : f32 {llvm.sret = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.sret attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.sret = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.nest attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_nest(%arg0 : f32 {llvm.nest}) -> f32 { - llvm.return %arg0 : f32 -} -// ----- - -// expected-error @below{{llvm.byval attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_byval(%arg0 : f32 {llvm.byval = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.byval attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.byval = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.byref attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_byval(%arg0 : f32 {llvm.byref = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.byref attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.byref = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.inalloca attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_byval(%arg0 : f32 {llvm.inalloca = f32}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.inalloca attribute attached to LLVM pointer argument of a different type}} -llvm.func @invalid_sret(%arg0 : !llvm.ptr {llvm.inalloca = i32}) -> !llvm.ptr { - llvm.return %arg0 : !llvm.ptr -} - -// ----- - -// expected-error @below{{llvm.align attribute attached to LLVM non-pointer argument}} -llvm.func @invalid_align(%arg0 : f32 {llvm.align = 4}) -> f32 { - llvm.return %arg0 : f32 -} - -// ----- - -// expected-error @below{{llvm.signext attribute attached to LLVM non-integer argument}} -llvm.func @invalid_signext(%arg0: f32 {llvm.signext}) { - "llvm.return"() : () -> () -} - -// ----- - -// expected-error @below{{llvm.zeroext attribute attached to LLVM non-integer argument}} -llvm.func @invalid_zeroext(%arg0: f32 {llvm.zeroext}) { - "llvm.return"() : () -> () -} - -// ----- - llvm.func @no_non_complex_struct() -> !llvm.array<2 x array<2 x array<2 x struct<(i32)>>>> { // expected-error @below{{expected struct type to be a complex number}} %0 = llvm.mlir.constant(dense<[[[1, 2], [3, 4]], [[42, 43], [44, 45]]]> : tensor<2x2x2xi32>) : !llvm.array<2 x array<2 x array<2 x struct<(i32)>>>> @@ -181,7 +91,7 @@ llvm.func @binary_int_intr_wrong_type(%arg0 : i32, %arg1 : f32) -> f32 { // ----- llvm.func @ternary_float_intr_wrong_type(%arg0 : f32, %arg1 : f32, %arg2 : i32) -> f32 { - // expected-error @below{{op operand #2 must be floating-point or LLVM dialect-compatible vector of floating-point}} + // expected-error @below{{op operand #2 must be floating point LLVM type or LLVM dialect-compatible vector of floating point LLVM type}} %0 = "llvm.intr.fma"(%arg0, %arg1, %arg2) : (f32, f32, i32) -> f32 llvm.return %0 : f32 } @@ -348,10 +258,3 @@ llvm.func @stepvector_intr_wrong_type() -> vector<7xf32> { %0 = llvm.intr.experimental.stepvector : vector<7xf32> llvm.return %0 : vector<7xf32> } - -// ----- - -// expected-error @below{{llvm.readonly attribute attached to LLVM non-pointer argument}} -llvm.func @wrong_readonly_attribute(%vec : f32 {llvm.readonly}) { - llvm.return -} diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index dde4bb3d95e7..a58dfde99b05 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1008,16 +1008,16 @@ llvm.func @gep(%ptr: !llvm.ptr)>>, %idx: i64, // CHECK-LABEL: define void @indirect_const_call(i64 {{%.*}}) llvm.func @indirect_const_call(%arg0: i64) { // CHECK-NEXT: call void @body(i64 %0) - %0 = llvm.mlir.addressof @body : !llvm.ptr> - llvm.call %0(%arg0) : (i64) -> () + %0 = llvm.mlir.addressof @body : !llvm.ptr + llvm.call %0(%arg0) : !llvm.ptr, (i64) -> () // CHECK-NEXT: ret void llvm.return } -// CHECK-LABEL: define i32 @indirect_call(ptr {{%.*}}, float {{%.*}}) -llvm.func @indirect_call(%arg0: !llvm.ptr>, %arg1: f32) -> i32 { -// CHECK-NEXT: %3 = call i32 %0(float %1) - %0 = llvm.call %arg0(%arg1) : (f32) -> i32 +// CHECK-LABEL: define i32 @indirect_call(ptr addrspace(42) {{%.*}}, float {{%.*}}) +llvm.func @indirect_call(%arg0: !llvm.ptr<42>, %arg1: f32) -> i32 { +// CHECK-NEXT: %3 = call addrspace(42) i32 %0(float %1) + %0 = llvm.call %arg0(%arg1) : !llvm.ptr<42>, (f32) -> i32 // CHECK-NEXT: ret i32 %3 llvm.return %0 : i32 } @@ -1124,19 +1124,75 @@ llvm.func @zeroextattr(%arg0: i1 {llvm.zeroext}) { // CHECK-LABEL: declare void @zeroextattr_decl(i1 zeroext) llvm.func @zeroextattr_decl(i1 {llvm.zeroext}) +// CHECK-LABEL: declare void @alignattr_decl(ptr align 64) +llvm.func @alignattr_decl(!llvm.ptr {llvm.align = 64 : i64}) + +// CHECK-LABEL: declare void @dereferenceableattr_decl(ptr dereferenceable(32)) +llvm.func @dereferenceableattr_decl(!llvm.ptr {llvm.dereferenceable = 32 : i64}) + +// CHECK-LABEL: declare void @dereferenceableornullattr_decl(ptr dereferenceable_or_null(32)) +llvm.func @dereferenceableornullattr_decl(!llvm.ptr {llvm.dereferenceable_or_null = 32 : i64}) + +// CHECK-LABEL: declare void @inregattr_decl(ptr inreg) +llvm.func @inregattr_decl(!llvm.ptr {llvm.inreg}) + +// CHECK-LABEL: declare void @nocaptureattr_decl(ptr nocapture) +llvm.func @nocaptureattr_decl(!llvm.ptr {llvm.nocapture}) + +// CHECK-LABEL: declare void @nofreeattr_decl(ptr nofree) +llvm.func @nofreeattr_decl(!llvm.ptr {llvm.nofree}) + +// CHECK-LABEL: declare void @nonnullattr_decl(ptr nonnull) +llvm.func @nonnullattr_decl(!llvm.ptr {llvm.nonnull}) + +// CHECK-LABEL: declare void @preallocatedattr_decl(ptr preallocated(float)) +llvm.func @preallocatedattr_decl(!llvm.ptr {llvm.preallocated = f32}) + +// CHECK-LABEL: declare ptr @returnedattr_decl(ptr returned) +llvm.func @returnedattr_decl(!llvm.ptr {llvm.returned}) -> !llvm.ptr + +// CHECK-LABEL: declare void @alignstackattr_decl(ptr alignstack(32)) +llvm.func @alignstackattr_decl(!llvm.ptr {llvm.alignstack = 32 : i64}) + +// CHECK-LABEL: declare void @writeonlyattr_decl(ptr writeonly) +llvm.func @writeonlyattr_decl(!llvm.ptr {llvm.writeonly}) + // CHECK-LABEL: declare align 4 ptr @alignattr_ret_decl() llvm.func @alignattr_ret_decl() -> (!llvm.ptr {llvm.align = 4}) + // CHECK-LABEL: declare noalias ptr @noaliasattr_ret_decl() llvm.func @noaliasattr_ret_decl() -> (!llvm.ptr {llvm.noalias}) + // CHECK-LABEL: declare noundef ptr @noundefattr_ret_decl() llvm.func @noundefattr_ret_decl() -> (!llvm.ptr {llvm.noundef}) + // CHECK-LABEL: declare signext i1 @signextattr_ret_decl() llvm.func @signextattr_ret_decl() -> (i1 {llvm.signext}) + // CHECK-LABEL: declare zeroext i1 @zeroextattr_ret_decl() llvm.func @zeroextattr_ret_decl() -> (i1 {llvm.zeroext}) -// CHECK-LABEL: @llvm_varargs(...) -llvm.func @llvm_varargs(...) +// CHECK-LABEL: declare nonnull ptr @nonnullattr_ret_decl() +llvm.func @nonnullattr_ret_decl() -> (!llvm.ptr {llvm.nonnull}) + +// CHECK-LABEL: declare dereferenceable(32) ptr @dereferenceableattr_ret_decl() +llvm.func @dereferenceableattr_ret_decl() -> (!llvm.ptr {llvm.dereferenceable = 32 : i64}) + +// CHECK-LABEL: declare dereferenceable_or_null(16) ptr @dereferenceableornullattr_ret_decl() +llvm.func @dereferenceableornullattr_ret_decl() -> (!llvm.ptr {llvm.dereferenceable_or_null = 16 : i64}) + +// CHECK-LABEL: declare inreg ptr @inregattr_ret_decl() +llvm.func @inregattr_ret_decl() -> (!llvm.ptr {llvm.inreg}) + +// CHECK-LABEL: @varargs(...) +llvm.func @varargs(...) + +// CHECK-LABEL: define void @varargs_call +llvm.func @varargs_call(%arg0 : i32) { +// CHECK: call void (...) @varargs(i32 %{{.*}}) + llvm.call @varargs(%arg0) : (i32) -> () + llvm.return +} llvm.func @intpointerconversion(%arg0 : i32) -> i32 { // CHECK: %2 = inttoptr i32 %0 to ptr @@ -1195,6 +1251,20 @@ llvm.func @complexintconstantarray() -> !llvm.array<2 x !llvm.array<2 x !llvm.st llvm.return %1 : !llvm.array<2 x !llvm.array<2 x !llvm.struct<(i32, i32)>>> } +// CHECK-LABEL: @indexconstantsplat +llvm.func @indexconstantsplat() -> vector<3xi32> { + %1 = llvm.mlir.constant(dense<42> : vector<3xindex>) : vector<3xi32> + // CHECK: ret <3 x i32> + llvm.return %1 : vector<3xi32> +} + +// CHECK-LABEL: @indexconstantarray +llvm.func @indexconstantarray() -> vector<3xi32> { + %1 = llvm.mlir.constant(dense<[0, 1, 2]> : vector<3xindex>) : vector<3xi32> + // CHECK: ret <3 x i32> + llvm.return %1 : vector<3xi32> +} + llvm.func @noreach() { // CHECK: unreachable llvm.unreachable @@ -1347,42 +1417,60 @@ llvm.func @atomicrmw( %f32_ptr : !llvm.ptr, %f32 : f32, %i32_ptr : !llvm.ptr, %i32 : i32) { // CHECK: atomicrmw fadd ptr %{{.*}}, float %{{.*}} monotonic - %0 = llvm.atomicrmw fadd %f32_ptr, %f32 monotonic : f32 + %0 = llvm.atomicrmw fadd %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw fsub ptr %{{.*}}, float %{{.*}} monotonic - %1 = llvm.atomicrmw fsub %f32_ptr, %f32 monotonic : f32 + %1 = llvm.atomicrmw fsub %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + // CHECK: atomicrmw fmax ptr %{{.*}}, float %{{.*}} monotonic + %2 = llvm.atomicrmw fmax %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + // CHECK: atomicrmw fmin ptr %{{.*}}, float %{{.*}} monotonic + %3 = llvm.atomicrmw fmin %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw xchg ptr %{{.*}}, float %{{.*}} monotonic - %2 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : f32 + %4 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw add ptr %{{.*}}, i32 %{{.*}} acquire - %3 = llvm.atomicrmw add %i32_ptr, %i32 acquire : i32 + %5 = llvm.atomicrmw add %i32_ptr, %i32 acquire : !llvm.ptr, i32 // CHECK: atomicrmw sub ptr %{{.*}}, i32 %{{.*}} release - %4 = llvm.atomicrmw sub %i32_ptr, %i32 release : i32 + %6 = llvm.atomicrmw sub %i32_ptr, %i32 release : !llvm.ptr, i32 // CHECK: atomicrmw and ptr %{{.*}}, i32 %{{.*}} acq_rel - %5 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : i32 + %7 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : !llvm.ptr, i32 // CHECK: atomicrmw nand ptr %{{.*}}, i32 %{{.*}} seq_cst - %6 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : i32 + %8 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : !llvm.ptr, i32 // CHECK: atomicrmw or ptr %{{.*}}, i32 %{{.*}} monotonic - %7 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : i32 + %9 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw xor ptr %{{.*}}, i32 %{{.*}} monotonic - %8 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : i32 + %10 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw max ptr %{{.*}}, i32 %{{.*}} monotonic - %9 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : i32 + %11 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw min ptr %{{.*}}, i32 %{{.*}} monotonic - %10 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : i32 + %12 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umax ptr %{{.*}}, i32 %{{.*}} monotonic - %11 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : i32 + %13 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umin ptr %{{.*}}, i32 %{{.*}} monotonic - %12 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : i32 + %14 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + // CHECK: atomicrmw uinc_wrap ptr %{{.*}}, i32 %{{.*}} monotonic + %15 = llvm.atomicrmw uinc_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + // CHECK: atomicrmw udec_wrap ptr %{{.*}}, i32 %{{.*}} monotonic + %16 = llvm.atomicrmw udec_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + + // CHECK: atomicrmw volatile + // CHECK-SAME: syncscope("singlethread") + // CHECK-SAME: align 8 + %17 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32 llvm.return } // CHECK-LABEL: @cmpxchg llvm.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %val: i32) { // CHECK: cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} acq_rel monotonic - %0 = llvm.cmpxchg %ptr, %cmp, %val acq_rel monotonic : i32 + %0 = llvm.cmpxchg %ptr, %cmp, %val acq_rel monotonic : !llvm.ptr, i32 // CHECK: %{{[0-9]+}} = extractvalue { i32, i1 } %{{[0-9]+}}, 0 %1 = llvm.extractvalue %0[0] : !llvm.struct<(i32, i1)> // CHECK: %{{[0-9]+}} = extractvalue { i32, i1 } %{{[0-9]+}}, 1 %2 = llvm.extractvalue %0[1] : !llvm.struct<(i32, i1)> + + // CHECK: cmpxchg weak volatile + // CHECK-SAME: syncscope("singlethread") + // CHECK-SAME: align 8 + %3 = llvm.cmpxchg weak volatile %ptr, %cmp, %val syncscope("singlethread") acq_rel monotonic {alignment = 8 : i64} : !llvm.ptr, i32 llvm.return } @@ -1692,6 +1780,25 @@ llvm.func @nontemporal_store_and_load() { // ----- +llvm.func @atomic_store_and_load(%ptr : !llvm.ptr) { + // CHECK: load atomic + // CHECK-SAME: acquire, align 4 + %1 = llvm.load %ptr atomic acquire {alignment = 4 : i64} : !llvm.ptr -> f32 + // CHECK: load atomic + // CHECK-SAME: syncscope("singlethread") acquire, align 4 + %2 = llvm.load %ptr atomic syncscope("singlethread") acquire {alignment = 4 : i64} : !llvm.ptr -> f32 + + // CHECK: store atomic + // CHECK-SAME: release, align 4 + llvm.store %1, %ptr atomic release {alignment = 4 : i64} : f32, !llvm.ptr + // CHECK: store atomic + // CHECK-SAME: syncscope("singlethread") release, align 4 + llvm.store %2, %ptr atomic syncscope("singlethread") release {alignment = 4 : i64} : f32, !llvm.ptr + llvm.return +} + +// ----- + // Check that the translation does not crash in absence of a data layout. module { // CHECK: declare void @module_default_layout @@ -1867,44 +1974,6 @@ llvm.func @switch_weights(%arg0: i32) -> i32 { // ----- -module { - llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { - %0 = llvm.mlir.constant(0 : i32) : i32 - %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) - llvm.br ^bb3(%0 : i32) - ^bb3(%1: i32): - %2 = llvm.icmp "slt" %1, %arg1 : i32 - // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] - llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb4: - %3 = llvm.add %1, %arg2 : i32 - // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] - %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr - // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] - llvm.br ^bb3(%3 : i32) {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts}} - ^bb5: - llvm.return - } - - llvm.metadata @metadata { - llvm.access_group @group1 - llvm.access_group @group2 - } -} - -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[PA_NODE:[0-9]+]], ![[UNROLL_DISABLE_NODE:[0-9]+]], ![[LICM_DISABLE_NODE:[0-9]+]], ![[INTERLEAVE_NODE:[0-9]+]], ![[PIPELINE_DISABLE_NODE:[0-9]+]], ![[II_NODE:[0-9]+]]} -// CHECK: ![[PA_NODE]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} -// CHECK: ![[GROUP_NODE1]] = distinct !{} -// CHECK: ![[GROUP_NODE2]] = distinct !{} -// CHECK: ![[UNROLL_DISABLE_NODE]] = !{!"llvm.loop.unroll.disable", i1 true} -// CHECK: ![[LICM_DISABLE_NODE]] = !{!"llvm.licm.disable", i1 true} -// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 1} -// CHECK: ![[PIPELINE_DISABLE_NODE]] = !{!"llvm.loop.pipeline.disable", i1 true} -// CHECK: ![[II_NODE]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} -// CHECK: ![[ACCESS_GROUPS_NODE]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]} - -// ----- - module { llvm.func @aliasScope(%arg1 : !llvm.ptr, %arg2 : !llvm.ptr, %arg3 : !llvm.ptr) { %0 = llvm.mlir.constant(0 : i32) : i32 diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir new file mode 100644 index 000000000000..c0ea4b7812c4 --- /dev/null +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -0,0 +1,278 @@ +// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s + +// CHECK-LABEL: @disableNonForced +llvm.func @disableNonForced() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} + +// ----- + +// CHECK-LABEL: @mustprogress +llvm.func @mustprogress() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.mustprogress"} + +// ----- + +// CHECK-LABEL: @isvectorized +llvm.func @isvectorized() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.isvectorized", i32 1} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @vectorizeOptions +llvm.func @vectorizeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.predicate.enable", i1 true} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.scalable.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.width", i32 16} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_vectorized", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_epilogue", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @interleaveOptions +llvm.func @interleaveOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[INTERLEAVE_NODE:[0-9]+]]} +// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 32} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @unrollOptions +llvm.func @unrollOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.count", i32 64} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.runtime.disable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_unrolled", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_remainder", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @unrollOptions2 +llvm.func @unrollOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unroll.enable"} +// CHECK-DAG: ![[VEC_NODE2:[0-9]+]] = !{!"llvm.loop.unroll.full"} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @unrollAndJamOptions +llvm.func @unrollAndJamOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.enable"} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.count", i32 8} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @licmOptions +llvm.func @licmOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.licm_versioning.disable"} + +// ----- + +// CHECK-LABEL: @licmOptions2 +llvm.func @licmOptions2() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.licm.disable"} + +// ----- + +#followup = #llvm.loop_annotation + +// CHECK-LABEL: @distributeOptions +llvm.func @distributeOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation + >} +^bb1: + llvm.return +} + +// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} +// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_coincident", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_sequential", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_fallback", ![[FOLLOWUP]]} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_all", ![[FOLLOWUP]]} + +// ----- + +// CHECK-LABEL: @pipelineOptions +llvm.func @pipelineOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 false} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 1} + +// ----- + +// CHECK-LABEL: @peeledOptions +llvm.func @peeledOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.peeled.count", i32 3} + +// ----- + +// CHECK-LABEL: @unswitchOptions +llvm.func @unswitchOptions() { + // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation>} +^bb1: + llvm.return +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}} +// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unswitch.partial.disable"} + +// ----- + +// CHECK-LABEL: @loopOptions +llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { + %0 = llvm.mlir.constant(0 : i32) : i32 + %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) + llvm.br ^bb3(%0 : i32) + ^bb3(%1: i32): + %2 = llvm.icmp "slt" %1, %arg1 : i32 + // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] + llvm.cond_br %2, ^bb4, ^bb5 {loop_annotation = #llvm.loop_annotation< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + ^bb4: + %3 = llvm.add %1, %arg2 : i32 + // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] + %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr + // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] + llvm.br ^bb3(%3 : i32) {loop_annotation = #llvm.loop_annotation< + licm = , + interleave = , + unroll = , pipeline = , + parallelAccesses = @metadata::@group1, @metadata::@group2>} + + ^bb5: + llvm.return +} + +llvm.metadata @metadata { + llvm.access_group @group1 + llvm.access_group @group2 +} + +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[PA_NODE:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} +// CHECK-DAG: ![[GROUP_NODE1:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[GROUP_NODE2:[0-9]+]] = distinct !{} +// CHECK-DAG: ![[UNROLL_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.unroll.disable"} +// CHECK-DAG: ![[LICM_DISABLE_NODE:[0-9]+]] = !{!"llvm.licm.disable"} +// CHECK-DAG: ![[INTERLEAVE_NODE:[0-9]+]] = !{!"llvm.loop.interleave.count", i32 1} +// CHECK-DAG: ![[PIPELINE_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 true} +// CHECK-DAG: ![[II_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2} +// CHECK-DAG: ![[ACCESS_GROUPS_NODE:[0-9]+]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]} diff --git a/mlir/test/Target/LLVMIR/openmp-reduction.mlir b/mlir/test/Target/LLVMIR/openmp-reduction.mlir index abeb46b1adc2..d66d65bd2845 100644 --- a/mlir/test/Target/LLVMIR/openmp-reduction.mlir +++ b/mlir/test/Target/LLVMIR/openmp-reduction.mlir @@ -17,7 +17,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -90,7 +90,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -178,7 +178,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -261,7 +261,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } @@ -340,7 +340,7 @@ combiner { atomic { ^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr): %2 = llvm.load %arg3 : !llvm.ptr - llvm.atomicrmw fadd %arg2, %2 monotonic : f32 + llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } diff --git a/mlir/test/Target/LLVMIR/tbaa.mlir b/mlir/test/Target/LLVMIR/tbaa.mlir index 26a96e47aee6..84f27be92326 100644 --- a/mlir/test/Target/LLVMIR/tbaa.mlir +++ b/mlir/test/Target/LLVMIR/tbaa.mlir @@ -16,11 +16,11 @@ module { %1 = llvm.mlir.constant(1 : i32) : i32 %2 = llvm.getelementptr inbounds %arg1[%0, 1] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg2_t", (i64, i64)> // CHECK: load i64, ptr %{{.*}},{{.*}}!tbaa ![[LTAG:[0-9]*]] - %3 = llvm.load %2 {llvm.tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> i64 + %3 = llvm.load %2 {tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> i64 %4 = llvm.trunc %3 : i64 to i32 %5 = llvm.getelementptr inbounds %arg0[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg1_t", (i32, i32)> // CHECK: store i32 %{{.*}}, ptr %{{.*}},{{.*}}!tbaa ![[STAG:[0-9]*]] - llvm.store %4, %5 {llvm.tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr + llvm.store %4, %5 {tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr llvm.return } } @@ -60,11 +60,11 @@ module { %1 = llvm.mlir.constant(1 : i32) : i32 %2 = llvm.getelementptr inbounds %arg1[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg2_t", (f32, f32)> // CHECK: load float, ptr %{{.*}},{{.*}}!tbaa ![[LTAG:[0-9]*]] - %3 = llvm.load %2 {llvm.tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> f32 + %3 = llvm.load %2 {tbaa = [@__tbaa::@tbaa_tag_4]} : !llvm.ptr -> f32 %4 = llvm.fptosi %3 : f32 to i32 %5 = llvm.getelementptr inbounds %arg0[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg1_t", (i32, i32)> // CHECK: store i32 %{{.*}}, ptr %{{.*}},{{.*}}!tbaa ![[STAG:[0-9]*]] - llvm.store %4, %5 {llvm.tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr + llvm.store %4, %5 {tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr llvm.return } } diff --git a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp index f863240f3465..f3506226b16c 100644 --- a/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp +++ b/mlir/test/lib/Dialect/LLVM/TestLowerToLLVM.cpp @@ -65,8 +65,7 @@ void buildTestLowerToLLVM(OpPassManager &pm, // Convert vector to LLVM (always needed). pm.addPass(createConvertVectorToLLVMPass( // TODO: add more options on a per-need basis. - LowerVectorToLLVMOptions().enableReassociateFPReductions( - options.reassociateFPReductions))); + ConvertVectorToLLVMPassOptions{options.reassociateFPReductions})); // Convert Math to LLVM (always needed). pm.addNestedPass(createConvertMathToLLVMPass()); // Expand complicated MemRef operations before lowering them. @@ -74,7 +73,7 @@ void buildTestLowerToLLVM(OpPassManager &pm, // The expansion may create affine expressions. Get rid of them. pm.addPass(createLowerAffinePass()); // Convert MemRef to LLVM (always needed). - pm.addPass(createMemRefToLLVMConversionPass()); + pm.addPass(createFinalizeMemRefToLLVMConversionPass()); // Convert Func to LLVM (always needed). pm.addPass(createConvertFuncToLLVMPass()); // Convert Index to LLVM (always needed). diff --git a/mlir/test/mlir-cpu-runner/async-value.mlir b/mlir/test/mlir-cpu-runner/async-value.mlir index cafdc9f57a7a..c9e0b8c9dc8b 100644 --- a/mlir/test/mlir-cpu-runner/async-value.mlir +++ b/mlir/test/mlir-cpu-runner/async-value.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-arith-to-llvm),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-arith-to-llvm),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner \ // RUN: -e main -entry-point-result=void -O0 \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ diff --git a/mlir/test/mlir-cpu-runner/async.mlir b/mlir/test/mlir-cpu-runner/async.mlir index 8f387121ad41..e63b878903de 100644 --- a/mlir/test/mlir-cpu-runner/async.mlir +++ b/mlir/test/mlir-cpu-runner/async.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,convert-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,finalize-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner \ // RUN: -e main -entry-point-result=void -O0 \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ diff --git a/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir b/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir index 3720402f8cae..ab14b17e16ec 100644 --- a/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir +++ b/mlir/test/mlir-cpu-runner/bare-ptr-call-conv.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm{use-bare-ptr-memref-call-conv=1},reconcile-unrealized-casts)" | mlir-cpu-runner -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext -entry-point-result=void | FileCheck %s +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm{use-bare-ptr-memref-call-conv=1},reconcile-unrealized-casts)" | mlir-cpu-runner -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext -entry-point-result=void | FileCheck %s // Verify bare pointer memref calling convention. `simple_add1_add2_test` // gets two 2xf32 memrefs, adds 1.0f to the first one and 2.0f to the second diff --git a/mlir/test/mlir-cpu-runner/copy.mlir b/mlir/test/mlir-cpu-runner/copy.mlir index d1769dccef3e..742fd5018ce7 100644 --- a/mlir/test/mlir-cpu-runner/copy.mlir +++ b/mlir/test/mlir-cpu-runner/copy.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/global-memref.mlir b/mlir/test/mlir-cpu-runner/global-memref.mlir index 937d8d3afb24..5a75781781c3 100644 --- a/mlir/test/mlir-cpu-runner/global-memref.mlir +++ b/mlir/test/mlir-cpu-runner/global-memref.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s func.func private @printMemrefF32(memref<*xf32>) attributes { llvm.emit_c_interface } func.func private @printMemrefI32(memref<*xi32>) attributes { llvm.emit_c_interface } diff --git a/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir b/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir index 134a1293d28c..685af8d3e063 100644 --- a/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir +++ b/mlir/test/mlir-cpu-runner/memref-reinterpret-cast.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf),convert-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf),finalize-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/memref-reshape.mlir b/mlir/test/mlir-cpu-runner/memref-reshape.mlir index 47ee21a7784a..94fe7d499b4e 100644 --- a/mlir/test/mlir-cpu-runner/memref-reshape.mlir +++ b/mlir/test/mlir-cpu-runner/memref-reshape.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,memref-expand,convert-arith-to-llvm),convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-scf-to-cf,memref-expand,convert-arith-to-llvm),finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext \ // RUN: | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir b/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir index 8a427ddf4b39..ccfea1cfedba 100644 --- a/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir +++ b/mlir/test/mlir-cpu-runner/sgemm-naive-codegen.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,lower-affine,convert-scf-to-cf,convert-arith-to-llvm),convert-vector-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" %s | mlir-cpu-runner -O3 -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s +// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,lower-affine,convert-scf-to-cf,convert-arith-to-llvm),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" %s | mlir-cpu-runner -O3 -e main -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s func.func @main() { %A = memref.alloc() : memref<16x16xf32> diff --git a/mlir/test/mlir-cpu-runner/unranked-memref.mlir b/mlir/test/mlir-cpu-runner/unranked-memref.mlir index f8a37f12c051..24562030aeea 100644 --- a/mlir/test/mlir-cpu-runner/unranked-memref.mlir +++ b/mlir/test/mlir-cpu-runner/unranked-memref.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ // RUN: mlir-cpu-runner -e main -entry-point-result=void \ // RUN: -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext,%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s diff --git a/mlir/test/mlir-cpu-runner/utils.mlir b/mlir/test/mlir-cpu-runner/utils.mlir index 565495b802f2..d902430c887a 100644 --- a/mlir/test/mlir-cpu-runner/utils.mlir +++ b/mlir/test/mlir-cpu-runner/utils.mlir @@ -1,7 +1,7 @@ -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_0d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-0D -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_1d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-1D -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_3d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-3D -// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e vector_splat_2d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-VECTOR-SPLAT-2D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_0d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-0D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_1d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-1D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e print_3d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-3D +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-linalg-to-loops,convert-scf-to-cf,convert-arith-to-llvm),convert-linalg-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | mlir-cpu-runner -e vector_splat_2d -entry-point-result=void -shared-libs=%mlir_lib_dir/libmlir_runner_utils%shlibext -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | FileCheck %s --check-prefix=PRINT-VECTOR-SPLAT-2D func.func @print_0d() { %f = arith.constant 2.00000e+00 : f32 diff --git a/mlir/test/mlir-opt/async.mlir b/mlir/test/mlir-opt/async.mlir index d5eafd1c6aef..b1c16b794d8b 100644 --- a/mlir/test/mlir-opt/async.mlir +++ b/mlir/test/mlir-opt/async.mlir @@ -1,6 +1,6 @@ // Check if mlir marks the corresponding function with required coroutine attribute. // -// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,convert-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ +// RUN: mlir-opt %s -pass-pipeline="builtin.module(async-to-async-runtime,func.func(async-runtime-ref-counting,async-runtime-ref-counting-opt),convert-async-to-llvm,func.func(convert-linalg-to-loops,convert-scf-to-cf),convert-linalg-to-llvm,finalize-memref-to-llvm,func.func(convert-arith-to-llvm),convert-func-to-llvm,reconcile-unrealized-casts)" \ // RUN: | FileCheck %s // CHECK: llvm.func @async_execute_fn{{.*}}attributes{{.*}}presplitcoroutine diff --git a/mlir/test/python/execution_engine.py b/mlir/test/python/execution_engine.py index 7b7ee953ea19..c40d522b8ca2 100644 --- a/mlir/test/python/execution_engine.py +++ b/mlir/test/python/execution_engine.py @@ -63,7 +63,7 @@ def testInvalidModule(): def lowerToLLVM(module): pm = PassManager.parse( - "builtin.module(convert-complex-to-llvm,convert-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)") + "builtin.module(convert-complex-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)") pm.run(module) return module diff --git a/mlir/test/python/integration/dialects/linalg/opsrun.py b/mlir/test/python/integration/dialects/linalg/opsrun.py index 585741ae9336..2a890eafa80f 100644 --- a/mlir/test/python/integration/dialects/linalg/opsrun.py +++ b/mlir/test/python/integration/dialects/linalg/opsrun.py @@ -199,7 +199,7 @@ def transform(module, boilerplate): pm.add("func.func(arith-expand)") pm.add("func.func(memref-expand)") pm.add("convert-vector-to-llvm") - pm.add("convert-memref-to-llvm") + pm.add("finalize-memref-to-llvm") pm.add("convert-func-to-llvm") pm.add("reconcile-unrealized-casts") pm.run(mod) diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp index 1aa5232f9671..926ebdf20d78 100644 --- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp +++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp @@ -365,6 +365,18 @@ class LLVMEnumAttr : public tblgen::EnumAttr { return cases; } + + std::vector getAllUnsupportedCases() const { + const auto *inits = def->getValueAsListInit("unsupported"); + + std::vector cases; + cases.reserve(inits->size()); + + for (const llvm::Init *init : *inits) + cases.emplace_back(cast(init)); + + return cases; + } }; // Wraper class around a Tablegen definition of a C-style LLVM enum attribute. @@ -473,6 +485,12 @@ static void emitOneEnumFromConversion(const llvm::Record *record, os << formatv(" return {0}::{1}::{2};\n", cppNamespace, cppClassName, cppEnumerant); } + for (const auto &enumerant : enumAttr.getAllUnsupportedCases()) { + StringRef llvmEnumerant = enumerant.getLLVMEnumerant(); + os << formatv(" case {0}::{1}:\n", llvmClass, llvmEnumerant); + os << formatv(" llvm_unreachable(\"unsupported case {0}::{1}\");\n", + enumAttr.getLLVMClassName(), llvmEnumerant); + } os << " }\n"; os << formatv(" llvm_unreachable(\"unknown {0} type\");", diff --git a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp index d52621a45075..d939b1aa65f3 100644 --- a/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp +++ b/mlir/tools/mlir-vulkan-runner/mlir-vulkan-runner.cpp @@ -69,11 +69,13 @@ static LogicalResult runMLIRPasses(Operation *op, modulePM.addPass(spirv::createSPIRVWebGPUPreparePass()); passManager.addPass(createConvertGpuLaunchFuncToVulkanLaunchFuncPass()); - LowerToLLVMOptions llvmOptions(module.getContext(), DataLayout(module)); passManager.addPass(createMemRefToLLVMConversionPass()); passManager.addPass(createConvertVectorToLLVMPass()); passManager.nest().addPass(LLVM::createRequestCWrappersPass()); - passManager.addPass(createConvertFuncToLLVMPass(llvmOptions)); + ConvertFuncToLLVMPassOptions funcToLLVMOptions{}; + funcToLLVMOptions.indexBitwidth = + DataLayout(module).getTypeSizeInBits(IndexType::get(module.getContext())); + passManager.addPass(createConvertFuncToLLVMPass(funcToLLVMOptions)); passManager.addPass(createReconcileUnrealizedCastsPass()); passManager.addPass(createConvertVulkanLaunchFuncToVulkanCallsPass()); diff --git a/mlir/unittests/ExecutionEngine/Invoke.cpp b/mlir/unittests/ExecutionEngine/Invoke.cpp index 57a260bfc5c8..eeefbfe258a8 100644 --- a/mlir/unittests/ExecutionEngine/Invoke.cpp +++ b/mlir/unittests/ExecutionEngine/Invoke.cpp @@ -52,8 +52,8 @@ static struct LLVMInitializer { /// Simple conversion pipeline for the purpose of testing sources written in /// dialects lowering to LLVM Dialect. static LogicalResult lowerToLLVMDialect(ModuleOp module) { - PassManager pm(module.getContext()); - pm.addPass(mlir::createMemRefToLLVMConversionPass()); + PassManager pm(module->getContext()); + pm.addPass(mlir::createFinalizeMemRefToLLVMConversionPass()); pm.addNestedPass(mlir::createArithToLLVMConversionPass()); pm.addPass(mlir::createConvertFuncToLLVMPass()); pm.addPass(mlir::createReconcileUnrealizedCastsPass());